Add Example Application: Llama 3.2 (1 billion parameters)

.github/workflows/ci-lint.yml

@@ -28,6 +28,7 @@ jobs:
         if: success() || failure()
         shell: bash
         run: |
+          reuse download --all
           reuse lint
 
       - name: Format Python

.github/workflows/test-examples.yml

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Test Example Applications
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+      - devel
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Test:
+    runs-on: [self-hosted, docker]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        env:
+          HOME: /workspace
+
+      - name: Prerequisites
+        uses: ./.github/actions/prereqs
+        with:
+          env_name: ci_env
+
+      - name: Install additional requirements for examples
+        shell: bash
+        env:
+          HOME: /workspace
+        run: |
+          set -euxo pipefail
+          source ci_env/bin/activate
+          pip install -r requirements_examples.txt
+
+      - name: Build examples
+        uses: ./.github/actions/build
+        with:
+          env_name: ci_env
+          cmake_extra_flags: "-DBUILD_APPLICATIONS=ON -DBUILD_EXAMPLES=OFF"
+
+      - name: Run examples
+        id: test
+        continue-on-error: true
+        uses: ./.github/actions/test
+        with:
+          env_name: ci_env
+          test_flags: "--log tests.log --csv-latest tests_latest.csv"
+
+      - name: Commit example test results
+        uses: ./.github/actions/commit_results
+        id: commit_results
+        with:
+          dir: examples
+          pretty_flags: "--metric='TTFT (mean)' --metric='TPS (mean)' -o pretty.md"
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: results-examples
+          path: ${{ steps.commit_results.outputs.results_dir }}
+          retention-days: 14
+
+      - name: Fail workflow if examples failed
+        if: steps.test.outcome == 'failure'
+        run: exit 1

CMakeLists.txt

@@ -60,8 +60,9 @@ endif()
 set(IRONCLAD_MLIR_AIE_RUNTIME_DIR "${IRONCLAD_MLIR_AIE_DIR}/runtime_lib/x86_64/test_lib/" CACHE PATH "Path to built/installed test_lib from MLIR_AIE")
 
 # More Options
-option(BUILD_EXAMPLES "Build examples" ON)
-option(EXTENSIVE_TESTING "Generate a larger number of examples and tests" OFF)
+option(BUILD_EXAMPLES "Build operators" ON)
+option(BUILD_APPLICATIONS "Build example applications" OFF)
+option(EXTENSIVE_TESTING "Generate a larger number of operators and tests" OFF)
 
 message(STATUS "IRONCLAD_PEANO_DIR: ${IRONCLAD_PEANO_DIR}")
 message(STATUS "IRONCLAD_MLIR_AIE_DIR: ${IRONCLAD_MLIR_AIE_DIR}")
@@ -95,8 +96,12 @@ add_subdirectory(aie_kernels)
 # Add examples
 # Pass the architecure to support aie2p
 if (BUILD_EXAMPLES)
-    include(cmake/IroncladExamples.cmake)
     add_subdirectory(example)
 endif()
 
+# Add example applications
+if (BUILD_APPLICATIONS)
+    add_subdirectory(applications)
+endif()
+
 generate_aie_ci_test_list("ci_tests.py")

LICENSES/CC0-1.0.txt

@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

REUSE.toml

@@ -0,0 +1,10 @@
+version = 1
+SPDX-PackageName = "iron"
+SPDX-PackageSupplier = "Advanced Micro Devices, Inc."
+SPDX-PackageDownloadLocation = "https://github.com/AARInternal/ironclad"
+
+[[annotations]]
+path = "applications/llama_3.2_1b/prompt.txt"
+precedence = "closest"
+SPDX-FileCopyrightText = "Public Domain"
+SPDX-License-Identifier = "CC0-1.0"

applications/CMakeLists.txt

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+add_subdirectory(llama_3.2_1b)

applications/llama_3.2_1b/CMakeLists.txt

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+add_aie_ci_test(llama_3.2_1b
+                RUN 
+                    "python3 ${CMAKE_CURRENT_SOURCE_DIR}/inference.py /srv/llama3.2-1b/model.safetensors /srv/llama3.2-1b/tokenizer.model --prompt_len 2048 --num_tokens 40"
+                CHECK
+                    "PASS!"
+                METRICS 
+                    "Total" [=[  Total time: (?P<metric>[\d\.e\+-]+) seconds]=]
+                    "TTFT" [=[  Prefill time: (?P<metric>[\d\.e\+-]+) seconds]=]
+                    "TPS" [=[  Tokens per second: (?P<metric>[\d\.e\+-]+)]=]
+                    "Num Tokens" [=[  Tokens generated: (?P<metric>[\d\.e\+-]+)]=]
+)

applications/llama_3.2_1b/README.md

@@ -0,0 +1,49 @@
+<!--
+SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Golden Model Inference
+
+## Model and Tokenizer Download Instructions
+
+To download the necessary files for the model, please follow the links below:
+
+- **Model File**: [model.safetensors](https://huggingface.co/meta-llama/Llama-3.2-1B/tree/main)
+- **Tokenizer File**: [tokenizer.model](https://huggingface.co/meta-llama/Llama-3.2-1B/tree/main/original)
+
+The CI tests expect these files to be placed in the directory `/srv/llama3.2-1b`;
+when calling `inference.py`, you will supply the paths to those two files on the command line, so you can place them anywhere.
+
+## Installation Instructions
+
+Before running `inference.py`, ensure you have the proper environment. To build the environment from scratch, follow the instructions below:
+
+1. Follow the IRON installation instructions in the repository root fist.
+   After this, you should have an `ironenv` environment set up and activated.
+
+2. Install the following additional requirements:
+   ```
+   python3 -m pip install -r requirements_examples.txt
+   ```
+
+## Running Inference
+
+Inference with Llama-3.2-1B can be run by specifying a number of tokens to generate based on a prompt. This is done with `inference.py`:  
+```bash  
+cd golden_model
+python inference.py /path/to/model.safetensors /path/to/tokenizer.model --num_tokens <NUM_TOKENS> --prompt <PROMPT>
+```
+
+`inference.py` has the following command format:  
+```bash
+python inference.py <weights_file_path> <tokenizer_file_path> [--num_tokens NUM_TOKENS] [--prompt PROMPT] [--use_prompt_template] [--save_outputs]
+```
+
+### Arguments:
+- `weights_file_path`: Path to the weights file (e.g., `model.safetensors`).
+- `tokenizer_file_path`: Path to the tokenizer file (e.g., `tokenizer.model`).
+- `--num_tokens`: (Optional) Number of tokens to predict. Default is `1`.
+- `--prompt`: (Optional) Prompt for the model to generate text from. Default is the text in `prompts.txt`.
+- `--use_prompt_template`: (Optional) Use a prompt template for the model. Should be passed in when using Instruct weights.
+- `--save_outputs`: (Optional) Enable hooks to save outputs of at each layer of the model.

applications/llama_3.2_1b/configs/llama32_1b.json

@@ -0,0 +1,36 @@
+{
+  "model_config": {
+    "vocab_size": 128256,
+    "context_length": 131072,
+    "emb_dim": 2048,
+    "n_heads": 32,
+    "n_layers": 16,
+    "hidden_dim": 8192,
+    "n_kv_groups": 8,
+    "use_kv_cache": true,
+    "use_aie_gemv": true,
+    "rope_base": 500000.0,
+    "dtype": "bfloat16",
+    "use_aie_final_norm": true,
+    "use_aie_ffn_gemm": true,
+    "use_aie_ffn_silu": true,
+    "use_aie_ffn_mul": true,
+    "use_aie_attn_projection_gemm": true,
+    "use_aie_rope": true,
+    "use_aie_norm1": true,
+    "use_aie_norm2": true,
+    "use_aie_residual": true,
+    "use_aie_regular_mha": false,
+    "use_aie_fused_mha": true,
+    "use_aie_final_gemm": false,
+    "rope_freq": {
+      "factor": 32.0,
+      "low_freq_factor": 1.0,
+      "high_freq_factor": 4.0,
+      "original_context_length": 8192
+    }
+  },
+  "aie_config": {
+    "device": "npu2"
+  }
+}

applications/llama_3.2_1b/configs/llama32_1b.json.license

@@ -0,0 +1,7 @@
+Copyright (c) Sebastian Raschka under Apache License 2.0.
+Source for "Build a Large Language Model From Scratch"
+  - https://www.manning.com/books/build-a-large-language-model-from-scratch
+Code: https://github.com/rasbt/LLMs-from-scratch/blob/main/ch05/07_gpt_to_llama/standalone-llama32.ipynb
+
+SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+SPDX-License-Identifier: Apache-2.0