From a3bd248e08797ef855859d3d76261c22cc1b6999 Mon Sep 17 00:00:00 2001 From: kingbri <8082010+kingbri1@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:50:30 -0400 Subject: [PATCH 1/4] Actions: Use GHCR as Docker layer cache Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com> --- .github/workflows/docker-image.yml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 7964670..3958b4a 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,5 +1,4 @@ name: Build and publish a Docker image -# Configures this workflow to run every time a change is pushed to the following branches. on: push: branches: ['main'] @@ -8,20 +7,17 @@ concurrency: group: "docker-${{ github.ref }}" cancel-in-progress: true -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +# Used for the Container registry domain and Docker image name env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: build-and-push-image: runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read packages: write - # strategy: matrix: include: @@ -30,31 +26,33 @@ jobs: - dockerfile: ./docker/Dockerfile.cu13 tag: cu13 steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - uses: actions/checkout@v4 + - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + + # Adds "latest" tag to the newly built image - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This is needed to add the `latest` tag to the newly built image tags: type=raw,value=${{ matrix.tag }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Builds and pushes the Docker image to GitHub Packages - name: Build and push Docker image uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 with: file: ${{ matrix.dockerfile }} - # target: production # Might need it in the future when we have multistage builds push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.tag }} + cache-to: type=gha,mode=max,scope=${{ matrix.tag }} From 7e4ccd5e8c68d9e7ca58d29cd2239a99ff9796d5 Mon Sep 17 00:00:00 2001 From: kingbri <8082010+kingbri1@users.noreply.github.com> Date: Fri, 26 Jun 2026 21:13:46 -0400 Subject: [PATCH 2/4] Actions: Point to GHCR cache instead of GHA cache Need a longer term cache storage Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com> --- .github/workflows/docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 3958b4a..dee575e 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -54,5 +54,5 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha,scope=${{ matrix.tag }} - cache-to: type=gha,mode=max,scope=${{ matrix.tag }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.tag }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.tag }},mode=max From 3cf468c28362c28be1c8fc731ce1ccaf7b2206d0 Mon Sep 17 00:00:00 2001 From: kingbri <8082010+kingbri1@users.noreply.github.com> Date: Fri, 26 Jun 2026 21:21:05 -0400 Subject: [PATCH 3/4] Actions: Fix docker buildx casing issue Add step to change the repo name to lowercase Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com> --- .github/workflows/docker-image.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index dee575e..0ff8832 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -10,7 +10,6 @@ concurrency: # Used for the Container registry domain and Docker image name env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} jobs: build-and-push-image: @@ -35,12 +34,16 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Lowercase image name + id: image + run: echo "name=$(echo "$GITHUB_REPOSITORY" | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT" + # Adds "latest" tag to the newly built image - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} tags: type=raw,value=${{ matrix.tag }} - name: Set up Docker Buildx @@ -54,5 +57,5 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.tag }} - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.tag }},mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:buildcache-${{ matrix.tag }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:buildcache-${{ matrix.tag }},mode=max From 2e1378c4d0b66243b9aecf1ce87885a3febe28c9 Mon Sep 17 00:00:00 2001 From: Ethan Jones <66649248+ethanj801@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:46:51 -0400 Subject: [PATCH 4/4] Fix chat completions returning null logprobs without reasoning/tool tags (#430) The content-span logprobs guard collected a token's logprobs only when `tag not in [t_think_end, t_tool_end]`. With no reasoning or tool tags configured both ends are None, and an ordinary content token also has tag None, so `None not in [None, None]` is False and logprobs are never collected. /v1/chat/completions then returns null logprobs for any model lacking both reasoning and tool tags, while /v1/completions returns them correctly. Filter unset tags out of the membership test, matching the `if s` filter the split regex above already applies to the same tags. --- endpoints/OAI/utils/chat_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/OAI/utils/chat_completion.py b/endpoints/OAI/utils/chat_completion.py index 4ff2bf0..69145c8 100644 --- a/endpoints/OAI/utils/chat_completion.py +++ b/endpoints/OAI/utils/chat_completion.py @@ -469,7 +469,7 @@ async def _chat_stream_collector( # out of a tag if ( "logprobs_content" in generation - and tag not in [t_think_end, t_tool_end] + and tag not in [t for t in (t_think_end, t_tool_end) if t] and not in_reasoning and not in_tool ):