chunked_publish_ts #3558
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Publish tree-sitter parsers | |
| on: | |
| repository_dispatch: | |
| types: [chunked_publish_ts] | |
| jobs: | |
| publish: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 50 # Don't DDOS the luarocks servers | |
| matrix: | |
| parsers: ${{ github.event.client_payload.parsers }} | |
| name: Pushing tree-sitter-${{ matrix.parsers.lang }} | |
| steps: | |
| - name: Clone parser repository | |
| run: | | |
| if [[ -n "${{ matrix.parsers.install_info.url }}" ]]; then | |
| REPO_URL=$(echo "${{ matrix.parsers.install_info.url }}" | sed 's/\.git$//') \ | |
| || REPO_URL="${{ matrix.parsers.install_info.url }}" | |
| echo "REPO_URL=$REPO_URL" >> $GITHUB_ENV | |
| if [[ ! "$REPO_URL" =~ ^https?://github\.com/ ]]; then | |
| GIT_SERVER_URL=$(echo "$REPO_URL" | sed -E 's/^(https?:\/\/[^\/]+).*/\1/') | |
| echo "GIT_SERVER_URL_OVERRIDE=$GIT_SERVER_URL" >> $GITHUB_ENV | |
| fi | |
| git clone "$REPO_URL.git" . || git clone "$REPO_URL" . | |
| else | |
| echo "REPO_URL=https://github.com/nvim-neorocks/luarocks-stub" >> $GITHUB_ENV | |
| fi | |
| - name: Get repo and parser metadata | |
| id: get-release | |
| run: | | |
| echo "PARSER_NAME=tree-sitter-${{ matrix.parsers.lang }}" >> $GITHUB_ENV | |
| echo "GITHUB_REPOSITORY_OVERRIDE=$(echo '${{ env.REPO_URL }}' | awk -F/ '{print $4"/"$5}')" >> $GITHUB_ENV | |
| - name: Install tree-sitter CLI | |
| uses: baptiste0928/cargo-install@v3 | |
| with: | |
| crate: tree-sitter-cli | |
| - uses: actions/setup-node@v4 | |
| if: ${{ matrix.parsers.install_info.generate }} | |
| - name: Install C/C++ Compiler | |
| uses: rlalik/setup-cpp-compiler@master | |
| with: | |
| compiler: clang-latest | |
| - name: Install Lua | |
| uses: leso-kn/gh-actions-lua@master | |
| with: | |
| luaVersion: "5.1" | |
| - name: Install Luarocks | |
| uses: hishamhm/gh-actions-luarocks@master | |
| - name: Get resources | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: | | |
| resources | |
| path: nurr | |
| - name: Clone nvim-treesitter and get revision | |
| run: | | |
| git clone --branch main "https://github.com/nvim-treesitter/nvim-treesitter.git" nvim-treesitter | |
| pushd nvim-treesitter | |
| git checkout "$(cat ../nurr/resources/nvim-treesitter.lock)" | |
| popd | |
| if [[ -n "${{ matrix.parsers.install_info.url }}" ]]; then | |
| PARSER_REVISION=$(jq -r ".\"${{ matrix.parsers.lang }}\".revision" nvim-treesitter/lockfile.json) | |
| else | |
| # Use stub | |
| PARSER_REVISION="548853648d7cff7e0d959ff95209e8aa97a793bc" | |
| fi | |
| echo "GITHUB_SHA_OVERRIDE=$PARSER_REVISION" >> $GITHUB_ENV | |
| - name: Determine specrev | |
| run: | | |
| luarocks download --dev $PARSER_NAME || true | |
| specrev=1 | |
| for existing_rockspec in $PARSER_NAME-*.rockspec; do | |
| if [[ -f $existing_rockspec ]]; then | |
| echo "Existing rockspec: $existing_rockspec" | |
| specrev_ext="${existing_rockspec##*-}" | |
| specrev="${specrev_ext%.*}" | |
| if grep -q $PARSER_REVISION $existing_rockspec; then | |
| specrev=$((specrev + 1)) | |
| fi | |
| fi | |
| done | |
| echo "SPECREV=$specrev" >> $GITHUB_ENV | |
| echo "specrev is $specrev" | |
| - name: Prepare Rockspec template | |
| run: | | |
| mv nurr/resources/tree-sitter-parser-rockspec.template rockspec.template | |
| sed -i 's/LANG/${{ matrix.parsers.lang }}/g' rockspec.template | |
| if [[ -z "${{ matrix.parsers.install_info.url }}" ]]; | |
| then | |
| sed -i 's/parser\s=\strue/parser = false/g' rockspec.template | |
| fi | |
| if [[ "${{ matrix.parsers.install_info.generate }}" == "true" ]]; | |
| then | |
| sed -i 's/generate\s=\sfalse/generate = true/g' rockspec.template | |
| fi | |
| if [[ "${{ matrix.parsers.install_info.generate_from_json }}" == "true" ]]; | |
| then | |
| sed -i 's/generate_from_json\s=\sfalse/generate_from_json = true/g' rockspec.template | |
| fi | |
| if [[ -n "${{ matrix.parsers.install_info.location }}" ]]; | |
| then | |
| sed -i 's@location\s=\snil@location = "${{ matrix.parsers.install_info.location }}"@g' rockspec.template | |
| fi | |
| if [[ "${{ env.REPO_URL }}" =~ ^https:\/\/git\.sr\.ht\/ ]]; then | |
| echo "fixing up template for sourcehut" | |
| sed -i "s/'\.zip'/'.tar.gz'/g" rockspec.template | |
| elif [[ "${{ env.REPO_URL }}" =~ ^https:\/\/gitlab\.com\/ ]]; then | |
| echo "fixing up template for gitlab" | |
| sed -i "s@'/archive/'@'/-/archive/'@g" rockspec.template | |
| fi | |
| QUERIES_DIR="nvim-treesitter/runtime/queries/${{ matrix.parsers.lang }}" | |
| for QUERY_FILE in "$QUERIES_DIR"/*; do | |
| BASE_NAME="$(basename "$QUERY_FILE")" | |
| # Injection string with file name and placeholder for file content | |
| INJECTION=" [\"$BASE_NAME\"] = [==[\nQUERY_CONTENT\n]==],\nQUERY_PLACEHOLDER" | |
| echo "Injecting placeholder for $BASE_NAME" | |
| sed -i "s/QUERY_PLACEHOLDER/$INJECTION/" rockspec.template | |
| echo "Injecting $BASE_NAME" | |
| sed -i -e "/QUERY_CONTENT/ {" -e "r $QUERY_FILE" -e "d" -e "}" rockspec.template | |
| done | |
| # Remove the final placeholder | |
| sed -i '/QUERY_PLACEHOLDER/d' rockspec.template | |
| # Inject nvim-treesitter license | |
| echo "Injecting nvim-treesitter license" | |
| sed -i -e "/NVIM_TREESITTER_LICENSE_CONTENT/ {" -e "r nvim-treesitter/LICENSE" -e "d" -e "}" rockspec.template | |
| cat rockspec.template | |
| - name: Prepare dev dependencies | |
| if: ${{ matrix.parsers.requires != null }} | |
| run: | | |
| DEV_DEPENDENCIES=$(echo -e "${{ join(matrix.parsers.requires, '\n') }}" | sed 's/^/tree-sitter-/') | |
| { | |
| echo 'DEV_DEPENDENCIES<<EOF' | |
| echo "$DEV_DEPENDENCIES" | |
| echo EOF | |
| } >> $GITHUB_ENV | |
| - name: Prepare versioned dependencies | |
| if: ${{ matrix.parsers.requires != null }} | |
| run: | | |
| DEPENDENCIES=$(echo -e "${{ join(matrix.parsers.requires, '\n') }}" | sed 's/^/tree-sitter-/') | |
| VERSIONED_DEPENDENCIES="" | |
| while read -r dep; do | |
| lang="${dep#tree-sitter-}" | |
| # Fetch the update count from the JSON file | |
| count=$(jq -r --arg lang "$lang" '.update_counts[] | select(.key == $lang) | .value' nurr/resources/ts-update-counts.json) | |
| if [[ -n "$count" ]]; then | |
| # If VERSIONED_DEPENDENCIES is not empty, append a newline before adding the next dependency | |
| [[ -n "$VERSIONED_DEPENDENCIES" ]] && VERSIONED_DEPENDENCIES+=$'\n' | |
| VERSIONED_DEPENDENCIES+="tree-sitter-$lang == 0.0.$count" | |
| else | |
| # Append the unchanged dependency | |
| [[ -n "$VERSIONED_DEPENDENCIES" ]] && VERSIONED_DEPENDENCIES+=$'\n' | |
| VERSIONED_DEPENDENCIES+="$dep" | |
| fi | |
| done <<< "$DEPENDENCIES" # Using <<< to pass the dependencies as input to the loop | |
| { | |
| echo 'VERSIONED_DEPENDENCIES<<EOF' | |
| echo "$VERSIONED_DEPENDENCIES" | |
| echo EOF | |
| } >> $GITHUB_ENV | |
| - name: Install `luarocks-build-treesitter-parser` Package | |
| run: | | |
| luarocks --verbose --local --lua-version=5.1 install luarocks-build-treesitter-parser | |
| - name: Install askalono-cli for license detection | |
| uses: baptiste0928/cargo-install@v3 | |
| with: | |
| crate: askalono-cli | |
| - name: Parser update count -> publish version | |
| run: | | |
| UPDATE_COUNT=$(jq -r --arg lang "${{ matrix.parsers.lang }}" '.update_counts[] | select(.key == $lang) | .value' ./nurr/resources/ts-update-counts.json) | |
| VERSION="0.0.$UPDATE_COUNT" | |
| echo "RELEASE_VERSION=$VERSION" >> $GITHUB_ENV | |
| echo "RELEASE_VERSION = $VERSION" | |
| - name: Check for published release | |
| run: | | |
| echo "PUBLISHED_COUNT=$(luarocks search ${{ env.PARSER_NAME }} ${{ env.RELEASE_VERSION }} --porcelain | grep '${{ env.PARSER_NAME }}\s+' | wc -l)" >> $GITHUB_ENV | |
| - name: Default to unknown license | |
| run: echo "LICENSE=UNKNOWN" >> $GITHUB_ENV | |
| - name: Get SPDX license | |
| continue-on-error: true | |
| run: | | |
| LICENSE=$(askalono crawl . | sed -n 's/License: \([^ ]*\).*/\1/p' || echo "UNKNOWN") | |
| LICENSE_COUNT=$(echo "$LICENSE" | grep -c .) | |
| if [ "$LICENSE_COUNT" -eq 1 ]; then | |
| echo "License: $LICENSE" | |
| else | |
| if [ "$LICENSE_COUNT" -eq 0 ]; then | |
| echo "WARNING: No license identified." | |
| else | |
| echo "WARNING: Multiple licenses identified." | |
| fi | |
| LICENSE="UNKNOWN" | |
| fi | |
| echo "LICENSE=$LICENSE" >> $GITHUB_ENV | |
| - name: Publish LuaRock (dev) | |
| uses: nvim-neorocks/luarocks-tag-release@v7 | |
| env: | |
| LUAROCKS_API_KEY: ${{ secrets.LUAROCKS_API_KEY }} | |
| with: | |
| name: ${{ env.PARSER_NAME }} | |
| version: 'scm' | |
| specrev: ${{ env.SPECREV }} | |
| summary: tree-sitter parser and Neovim queries for ${{ matrix.parsers.lang }} | |
| template: "rockspec.template" | |
| license: ${{ env.LICENSE }} | |
| labels: | | |
| neovim | |
| tree-sitter | |
| dependencies: ${{ env.DEV_DEPENDENCIES }} | |
| extra_luarocks_args: | | |
| --namespace=neorocks | |
| --force | |
| - name: Publish LuaRock (release) | |
| uses: nvim-neorocks/luarocks-tag-release@v7 | |
| if: ${{ env.PUBLISHED_COUNT == '0' }} | |
| env: | |
| LUAROCKS_API_KEY: ${{ secrets.LUAROCKS_API_KEY }} | |
| with: | |
| name: ${{ env.PARSER_NAME }} | |
| version: ${{ env.RELEASE_VERSION }} | |
| summary: tree-sitter parser for ${{ matrix.parsers.lang }} | |
| template: "rockspec.template" | |
| license: ${{ env.LICENSE }} | |
| labels: | | |
| neovim | |
| tree-sitter | |
| dependencies: ${{ env.VERSIONED_DEPENDENCIES }} | |
| extra_luarocks_args: | | |
| --namespace=neorocks |