Restructuring, getting rid of more memory leaks, and improving readability

.github/workflows/release.yaml

@@ -8,14 +8,11 @@ permissions:
 env:
   ARTIFACT-LINUX-X86_64-EXTENSION: sqlite-vss-linux-x86_64
   ARTIFACT-MACOS-X86_64-EXTENSION: sqlite-vss-macos-x86_64
-  ARTIFACT-MACOS-AARCH64-EXTENSION: sqlite-vss-macos-aarch64
   ARTIFACT-WINDOWS-X86_64-EXTENSION: sqlite-vss-windows-x86_64
-  ARTIFACT-LINUX-X86_64-WHEELS: sqlite-vss-linux-x86_64-wheels
-  ARTIFACT-MACOS-X86_64-WHEELS: sqlite-vss-macos-x86_64-wheels
-  ARTIFACT-MACOS-AARCH64-WHEELS: sqlite-vss-macos-aarch64-wheels
+  ARTIFACT-MACOS-AARCH64-EXTENSION: sqlite-vss-macos-aarch64
 jobs:
-  build-linux-x86_64-extension:
-    runs-on: ubuntu-20.04
+  build-macos-aarch64-extension:
+    runs-on: flyci-macos-large-latest-m1
     steps:
       - uses: actions/checkout@v3
         with:
@@ -30,33 +27,26 @@ jobs:
       - if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
         working-directory: vendor/sqlite
         run: ./configure && make
-
-      # TODO how cache this?
-      - run: sudo apt-get install -y cmake libgomp1
+      - run: brew install llvm
+      - id: cache-cmake-build
+        uses: actions/cache@v3
+        with:
+          path: build
+          key: ${{ runner.os }}-build
+      - run: make patch-openmp
       - run: make loadable-release static-release
+        env:
+          # `brew info libomp` gives the correct one, with .a file for static openmp builds
+          CC: /opt/homebrew/opt/llvm/bin/clang
+          CXX: /opt/homebrew/opt/llvm/bin/clang++
+          LDFLAGS: "-L/usr/local/opt/libomp/lib/"
+          CPPFLAGS: "-I/usr/local/opt/libomp/include/"
       - uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}
+          name: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}
           path: dist/release/*
-  build-linux-x86_64-python:
+  build-linux-x86_64-extension:
     runs-on: ubuntu-20.04
-    needs: [build-linux-x86_64-extension]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}
-          path: dist/release/
-      - uses: actions/setup-python@v3
-      - run: pip install wheel
-      - run: make python-release
-      - run: make datasette-release
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-LINUX-X86_64-WHEELS }}
-          path: dist/release/wheels/*.whl
-  build-macos-x86_64-extension:
-    runs-on: macos-latest
     steps:
       - uses: actions/checkout@v3
         with:
@@ -71,47 +61,20 @@ jobs:
       - if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
         working-directory: vendor/sqlite
         run: ./configure && make
-      - run: brew install llvm
-      - id: cache-cmake-build
-        uses: actions/cache@v3
-        with:
-          path: build
-          key: ${{ runner.os }}-build
-      - run: make patch-openmp
+
+      # TODO how cache this?
+      - run: sudo apt-get install -y cmake libgomp1
       - run: make loadable-release static-release
-        env:
-          # `brew info libomp` gives the correct one, with .a file for static openmp builds
-          CC: /usr/local/opt/llvm/bin/clang
-          CXX: /usr/local/opt/llvm/bin/clang++
-          LDFLAGS: "-L/usr/local/opt/libomp/lib/"
-          CPPFLAGS: "-I/usr/local/opt/libomp/include/"
       - uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}
+          name: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}
           path: dist/release/*
-  build-macos-x86_64-python:
+  build-macos-x86_64-extension:
     runs-on: macos-latest
-    needs: [build-macos-x86_64-extension]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}
-          path: dist/release/
-      - uses: actions/setup-python@v3
-      - run: pip install wheel
-      - run: make python-release
-      - run: make datasette-release
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-MACOS-X86_64-WHEELS }}
-          path: dist/release/wheels/*.whl
-  build-macos-aarch64-extension:
-    runs-on: [self-hosted, mm1]
     steps:
       - uses: actions/checkout@v3
         with:
-          submodules: "recursive"
+          submodules: recursive
       - id: cache-sqlite-build
         uses: actions/cache@v3
         with:
@@ -122,61 +85,24 @@ jobs:
       - if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
         working-directory: vendor/sqlite
         run: ./configure && make
+      - run: brew install llvm
+      - id: cache-cmake-build
+        uses: actions/cache@v3
+        with:
+          path: build
+          key: ${{ runner.os }}-build
       - run: make patch-openmp
       - run: make loadable-release static-release
         env:
           # `brew info libomp` gives the correct one, with .a file for static openmp builds
-          CC: /opt/homebrew/opt/llvm/bin/clang
-          CXX: /opt/homebrew/opt/llvm/bin/clang++
-          LDFLAGS: "-L/opt/homebrew/opt/libomp/lib"
-          CPPFLAGS: "-I/opt/homebrew/opt/libomp/include"
+          CC: /usr/local/opt/llvm/bin/clang
+          CXX: /usr/local/opt/llvm/bin/clang++
+          LDFLAGS: "-L/usr/local/opt/libomp/lib/"
+          CPPFLAGS: "-I/usr/local/opt/libomp/include/"
       - uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}
+          name: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}
           path: dist/release/*
-  build-macos-aarch64-python:
-    runs-on: [self-hosted, mm1]
-    needs: [build-macos-aarch64-extension]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}
-          path: dist/release/
-      - run: pip3 install wheel
-      - run: make python-release IS_MACOS_ARM=1
-      - run: make datasette-release
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-MACOS-AARCH64-WHEELS }}
-          path: dist/release/wheels/*.whl
-  upload-deno:
-    needs:
-      [
-        build-macos-x86_64-extension,
-        build-macos-aarch64-extension,
-        build-linux-x86_64-extension,
-      ]
-    permissions:
-      contents: write
-    runs-on: ubuntu-latest
-    outputs:
-      deno-checksums: ${{ steps.deno-assets.outputs.result }}
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v2
-      - id: deno-assets
-        uses: actions/github-script@v6
-        env:
-          ARTIFACT-LINUX-X86_64-EXTENSION: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}
-          ARTIFACT-MACOS-X86_64-EXTENSION: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}
-          ARTIFACT-MACOS-AARCH64-EXTENSION: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          result-encoding: string
-          script: |
-            const script = require('.github/workflows/upload-deno-assets.js')
-            return await script({github, context})
   upload-extensions:
     needs:
       [
@@ -201,164 +127,3 @@ jobs:
             macos-x86_64:     ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}/*
             macos-aarch64:    ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}/*
             linux-x86_64:     ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}/*
-  upload-checksums:
-    needs: [upload-extensions, upload-deno]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/github-script@v6
-        env:
-          CHECKSUMS: "${{ needs.upload-extensions.outputs.checksums }}\n${{ needs.upload-deno.outputs.deno-checksums }}"
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const { owner, repo } = context.repo;
-            const release = await github.rest.repos.getReleaseByTag({
-              owner,
-              repo,
-              tag: process.env.GITHUB_REF.replace("refs/tags/", ""),
-            });
-            const release_id = release.data.id;
-            github.rest.repos.uploadReleaseAsset({
-              owner,
-              repo,
-              release_id,
-              name: "checksums.txt",
-              data: process.env.CHECKSUMS,
-            });
-  upload-hex:
-    runs-on: ubuntu-latest
-    needs: [upload-extensions]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: erlef/setup-beam@v1
-        with:
-          otp-version: "24"
-          rebar3-version: "3.16.1"
-          elixir-version: "1.14"
-      - run: ./scripts/elixir_generate_checksum.sh "${{ needs.upload-extensions.outputs.checksums }}"
-      - run: mix deps.get
-        working-directory: ./bindings/elixir
-      - run: mix compile --docs
-        working-directory: ./bindings/elixir
-      - run: mix hex.publish --yes
-        working-directory: ./bindings/elixir
-        env:
-          HEX_API_KEY: ${{ secrets.HEX_API_KEY }}
-  upload-npm:
-    needs:
-      [
-        build-macos-x86_64-extension,
-        build-macos-aarch64-extension,
-        build-linux-x86_64-extension,
-      ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v2
-      - run: |
-          cp ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}/*.so bindings/node/sqlite-vss-linux-x64/lib/
-          cp ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}/*.dylib bindings/node/sqlite-vss-darwin-x64/lib/
-          cp ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}/*.dylib bindings/node/sqlite-vss-darwin-arm64/lib/
-      - uses: actions/setup-node@v3
-        with:
-          node-version: "16"
-          registry-url: "https://registry.npmjs.org"
-      - name: Publish NPM sqlite-vss-linux-x64
-        working-directory: bindings/node/sqlite-vss-linux-x64
-        run: npm publish --access public
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      - name: Publish NPM sqlite-vss-darwin-x64
-        working-directory: bindings/node/sqlite-vss-darwin-x64
-        run: npm publish --access public
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      - name: Publish NPM sqlite-vss-darwin-arm64
-        working-directory: bindings/node/sqlite-vss-darwin-arm64
-        run: npm publish --access public
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      - name: Publish NPM sqlite-vss
-        working-directory: bindings/node/sqlite-vss
-        run: npm publish --access public
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-  upload-pypi:
-    needs:
-      [
-        build-linux-x86_64-python,
-        build-macos-x86_64-python,
-        build-macos-aarch64-python,
-      ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-LINUX-X86_64-WHEELS }}
-          path: dist
-      - uses: actions/download-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-MACOS-X86_64-WHEELS }}
-          path: dist
-      - uses: actions/download-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT-MACOS-AARCH64-WHEELS }}
-          path: dist
-      - uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          skip-existing: true
-  upload-gem:
-    needs:
-      [
-        build-macos-x86_64-extension,
-        build-macos-aarch64-extension,
-        build-linux-x86_64-extension,
-      ]
-    permissions:
-      contents: write
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3.2
-      - run: |
-          rm bindings/ruby/lib/*.{dylib,so,dll} || true
-          cp ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}/*.dylib bindings/ruby/lib
-          gem -C bindings/ruby build -o x86_64-darwin.gem sqlite_vss.gemspec
-        env:
-          PLATFORM: x86_64-darwin
-      - run: |
-          rm bindings/ruby/lib/*.{dylib,so,dll} || true
-          cp ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}/*.dylib bindings/ruby/lib
-          gem -C bindings/ruby build -o arm64-darwin.gem sqlite_vss.gemspec
-        env:
-          PLATFORM: arm64-darwin
-      - run: |
-          rm bindings/ruby/lib/*.{dylib,so,dll} || true
-          cp ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}/*.so bindings/ruby/lib
-          gem -C bindings/ruby build -o x86_64-linux.gem sqlite_vss.gemspec
-        env:
-          PLATFORM: x86_64-linux
-      - run: |
-          gem push bindings/ruby/x86_64-darwin.gem
-          gem push bindings/ruby/arm64-darwin.gem
-          gem push bindings/ruby/x86_64-linux.gem
-        env:
-          GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}
-  upload-crate:
-    runs-on: ubuntu-latest
-    needs: [upload-extensions]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-      - run: cargo publish --no-verify
-        working-directory: ./bindings/rust
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}