Feature/misc fixes

.github/workflows/buildAndTest.yml

@@ -2,9 +2,9 @@ name: Build and Test
 
 on:
   pull_request:
-    branches: [ main ]
+    branches: [ feature/misc_fixes ]
   push:
-    branches: [ main ]
+    branches: [ feature/misc_fixes ]
   workflow_dispatch:
 
 # Ensure that only a single job or workflow using the same
@@ -25,9 +25,9 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os-arch: [ubuntu-x86_64, macos-arm64, windows-x86_64]
-        llvm-build: [in-tree, out-of-tree]
-        torch-binary: [ON, OFF]
+        os-arch: [ubuntu-x86_64] # macos-arm64, windows-x86_64
+        llvm-build: [in-tree] # out-of-tree
+        torch-binary: [ON] # OFF
         torch-version: [nightly, stable]
         exclude:
           # Exclude llvm in-tree and pytorch source
@@ -51,11 +51,11 @@ jobs:
         include:
           # Specify OS versions
           - os-arch: ubuntu-x86_64
-            os: a100
-          - os-arch: macos-arm64
-            os: macos-latest
-          - os-arch: windows-x86_64
-            os: windows-latest
+            os: ubuntu-latest # a100
+          #- os-arch: macos-arm64
+          #  os: macos-latest
+          #- os-arch: windows-x86_64
+          #  os: windows-latest
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -71,7 +71,6 @@ jobs:
       uses: actions/checkout@v3
       with:
         submodules: 'true'
-        fetch-depth: 0
 
     - name: Fetch PyTorch commit hash
       if: ${{ matrix.os-arch != 'windows-x86_64' }}

.github/workflows/buildRelease.yml

@@ -13,11 +13,16 @@ on:
 jobs:
   build_linux:
     name: Manylinux x86_64 Build
-    runs-on: a100
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: write
+      packages: write
     strategy:
       matrix:
-        package: [ torch-mlir, torch-mlir-core ]
-        py_version: [ cp38-cp38, cp311-cp311 ]
+        package: [ torch-mlir ]
+        py_version: [ cp38-cp38, cp310-cp310 ] # cp311-cp311
+        torch-version: [stable] # nightly
         exclude:
           - package: torch-mlir-core
             py_version: cp38-cp38
@@ -36,7 +41,6 @@ jobs:
       uses: actions/checkout@v3
       with:
         submodules: 'true'
-        fetch-depth: 0
 
     - uses: ./.github/actions/setup-build
       with:
@@ -46,7 +50,11 @@ jobs:
         cd $GITHUB_WORKSPACE
         TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }}
         printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version
-        TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} ./build_tools/python_deploy/build_linux_packages.sh
+        TM_SKIP_TESTS=ON \
+        TM_PYTHON_VERSIONS=${{ matrix.py_version }} \
+        TM_PACKAGES=${{ matrix.package }} \
+        TM_TORCH_VERSION="${{ matrix.torch-version }}" \
+        ./build_tools/python_deploy/build_linux_packages.sh
 
     # If we were given a release_id, then upload the package we just built
     # to the github releases page.
@@ -55,7 +63,7 @@ jobs:
       id: upload-release-assets
       uses: dwenegar/upload-release-assets@v1
       env:
-        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
       with:
         release_id: ${{ github.event.inputs.release_id }}
         assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl
@@ -66,7 +74,7 @@ jobs:
       id: publish_release
       uses: eregon/publish-release@v1
       env:
-        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
       with:
         release_id: ${{ github.event.inputs.release_id }}
     - name: Create dist directory
@@ -86,6 +94,7 @@ jobs:
         path: dist
 
   build_linux_arm64:
+    if: false
     name: Manylinux arm64 Build
     runs-on: linux-arm64
     strategy:
@@ -155,6 +164,7 @@ jobs:
         path: dist
 
   build_macos:
+    if: false
     name: MacOS Build
     runs-on: macos-latest
     strategy:
@@ -215,6 +225,7 @@ jobs:
         path: dist
 
   build_windows:
+    if: false
     name: Windows Build
     runs-on: windows-latest
     strategy:
@@ -286,11 +297,15 @@ jobs:
 
   publish_releases:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: write
+      packages: write
     needs:
     - build_linux
-    - build_linux_arm64
-    - build_macos
-    - build_windows
+    #- build_linux_arm64
+    #- build_macos
+    #- build_windows
 
     # Publish even if one of the builds failed
     if: ${{ always() }}
@@ -300,7 +315,7 @@ jobs:
       uses: benc-uk/workflow-dispatch@v1
       with:
         workflow: Publish releases page
-        token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+        token: ${{ secrets.GITHUB_TOKEN  }}
 
     # Wheels must be published from a linux environment.
     #

.github/workflows/gh-pages-releases.yml

@@ -8,9 +8,11 @@ jobs:
   scrape_and_publish_releases:
     name: "Scrape and publish releases"
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
 
     # Don't run this in everyone's forks.
-    if: github.repository == 'llvm/torch-mlir'
+    if: github.repository == 'xilinx/torch-mlir'
 
     steps:
       - name: Prepare workspace
@@ -20,10 +22,8 @@ jobs:
           sudo rm -rf $GITHUB_WORKSPACE/*
       - name: Checking out repository
         uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
       - name: Run scrape releases script
-        run: python ./build_tools/scrape_releases.py llvm torch-mlir > /tmp/index.html
+        run: python ./build_tools/scrape_releases.py xilinx torch-mlir > /tmp/index.html
         shell: bash
       - run: git fetch --all
       - run: git switch github-pages

.github/workflows/oneshotSnapshotPackage.yml

@@ -8,18 +8,19 @@ jobs:
     name: "Tag snapshot release"
     runs-on: ubuntu-latest
     # Don't run this in everyone's forks.
-    if: github.repository == 'llvm/torch-mlir'
+    #if: github.repository == 'llvm/torch-mlir'
     steps:
       - name: Prepare workspace
         run: |
           # Clear the workspace directory so that we don't run into errors about
           # existing lock files.
           sudo rm -rf $GITHUB_WORKSPACE/*
 
-      - name: Checking out repository
+      - name: Checkout torch-mlir
         uses: actions/checkout@v3
         with:
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+            submodules: 'true'
+            fetch-depth: 0
 
       - name: Compute version
         run: |

.github/workflows/releaseSnapshotPackage.yml

@@ -2,7 +2,7 @@ name: Release snapshot package
 
 on:
   schedule:
-    - cron: '0 11 * * *'
+    - cron: '17 4 * * *'
 
   workflow_dispatch:
 
@@ -11,7 +11,12 @@ jobs:
     name: "Tag snapshot release"
     runs-on: ubuntu-latest
     # Don't run this in everyone's forks.
-    if: github.repository == 'llvm/torch-mlir'
+    #if: github.repository == 'llvm/torch-mlir'
+    permissions:
+      contents: write
+      actions: write
+    env:
+        BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
     steps:
 
       - name: Prepare workspace
@@ -22,8 +27,6 @@ jobs:
 
       - name: Checking out repository
         uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
 
       - name: Compute version
         run: |
@@ -40,15 +43,15 @@ jobs:
       - name: Pushing changes
         uses: ad-m/[email protected]
         with:
-          github_token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
-          branch: main
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ env.BRANCH_NAME }} 
           tags: true
 
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
         with:
           tag_name: ${{ env.tag_name }}
           release_name: torch-mlir snapshot ${{ env.tag_name }}
@@ -57,17 +60,15 @@ jobs:
           draft: true
           prerelease: false
 
-      - name: "Invoke workflow :: Build and Test"
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: Build and Test
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
-          ref: "${{ env.tag_name }}"
+      # - name: "Invoke workflow :: Build and Test"
+      #   uses: benc-uk/workflow-dispatch@v1
+      #   with:
+      #     workflow: Build and Test
+      #     ref: "${{ env.tag_name }}"
 
       - name: "Invoke workflow :: Release Build"
         uses: benc-uk/workflow-dispatch@v1
         with:
           workflow: Release Build
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
           ref: "${{ env.tag_name }}"
           inputs: '{"release_id": "${{ steps.create_release.outputs.id }}", "python_package_version": "${{ env.package_version }}"}'

.gitmodules

@@ -1,6 +1,7 @@
 [submodule "externals/llvm-project"]
 	path = externals/llvm-project
-	url = https://github.com/llvm/llvm-project.git
+    url = https://github.com/Xilinx/llvm-project.git
+    branch = misc_fixes
 [submodule "externals/mlir-hlo"]
 	path = externals/mlir-hlo
 	url = https://github.com/tensorflow/mlir-hlo.git

build_tools/python_deploy/build_linux_packages.sh

@@ -116,9 +116,9 @@ function run_on_host() {
   docker run --rm \
     -v "${repo_root}:/main_checkout/torch-mlir" \
     -v "${TM_OUTPUT_DIR}:/wheelhouse" \
-    -v "${HOME}:/home/${USER}" \
+    -v "${PWD}:$PWD" \
     --user ${USERID}:${GROUPID} \
-    --workdir="/home/$USER" \
+    --workdir="$PWD" \
     --volume="/etc/group:/etc/group:ro" \
     --volume="/etc/passwd:/etc/passwd:ro" \
     --volume="/etc/shadow:/etc/shadow:ro" \
@@ -275,7 +275,6 @@ function test_in_tree() {
 
   cd /main_checkout/torch-mlir/
   export PYTHONPATH="/main_checkout/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir"
-
   case $torch_version in
     nightly)
       echo ":::: Test with nightly torch"
@@ -300,7 +299,10 @@ function test_in_tree() {
       exit 1
       ;;
     esac
-
+
+  echo ":::: Run make_fx + TOSA e2e integration tests"
+  python -m e2e_testing.main --config=make_fx_tosa -v
+
   echo ":::: Run TorchDynamo e2e integration tests"
   python -m e2e_testing.main --config=torchdynamo -v
 
@@ -422,7 +424,7 @@ function build_torch_mlir() {
       ;;
     stable)
       echo ":::: Using stable dependencies"
-      python3 -m pip install --no-cache-dir torch torchvision
+      python3 -m pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu
       python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/build-requirements.txt
       CMAKE_GENERATOR=Ninja \
       TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \

e2e_testing/main.py

@@ -29,6 +29,7 @@
 
 from .xfail_sets import (
     LINALG_XFAIL_SET,
+    MAKE_FX_TOSA_PASS_SET,
     STABLEHLO_PASS_SET,
     TOSA_PASS_SET,
     LTC_XFAIL_SET,
@@ -42,7 +43,7 @@
 register_all_tests()
 
 def _get_argparse():
-    config_choices = ["native_torch", "torchscript", "linalg", "stablehlo", "tosa", "lazy_tensor_core", "torchdynamo"]
+    config_choices = ["native_torch", "torchscript", "linalg", "stablehlo", "make_fx_tosa", "tosa", "lazy_tensor_core", "torchdynamo"]
     parser = argparse.ArgumentParser(description="Run torchscript e2e tests.")
     parser.add_argument("-c", "--config",
         choices=config_choices,
@@ -94,6 +95,10 @@ def main():
         config = TosaBackendTestConfig(LinalgOnTensorsTosaBackend())
         xfail_set = all_test_unique_names - TOSA_PASS_SET
         crashing_set = set()
+    elif args.config == "make_fx_tosa":
+        config = TosaBackendTestConfig(LinalgOnTensorsTosaBackend(), use_make_fx=True)
+        xfail_set = all_test_unique_names - MAKE_FX_TOSA_PASS_SET
+        crashing_set = set()
     elif args.config == "stablehlo":
         config = StablehloBackendTestConfig(LinalgOnTensorsStablehloBackend())
         xfail_set = all_test_unique_names - STABLEHLO_PASS_SET