Add input parameter for source repository cloning

Author: JohT

.github/workflows/internal-java-code-analysis.yml

@@ -46,9 +46,9 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }}
-      sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
       artifacts-upload-name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }}
       additional-maven-artifacts: ${{ steps.set-additional-maven-artifacts.outputs.additional-maven-artifacts }}
+      source-repository-branch: ${{ steps.set-source-repository-branch.outputs.source-repository-branch }}
 
     env: 
       PROJECT_NAME: AxonFramework
@@ -76,7 +76,7 @@ jobs:
         mkdir -p ${{ steps.set-analysis-name.outputs.analysis-name }}
         cd ${{ steps.set-analysis-name.outputs.analysis-name }}
         echo "Working directory: $( pwd -P )"
-        ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }}
+        ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }} --skip-clone
 
     - name: Debug folder structure in temp directory
       if: runner.debug == '1'
@@ -87,10 +87,6 @@ jobs:
     - name: (Prepare Code to Analyze) Generate ARTIFACT_UPLOAD_ID
       run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV
 
-    - name: (Prepare Code to Analyze) Set sources-upload-name
-      id: set-sources-upload-name
-      run: echo "sources-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-sources_input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
- 
     - name: (Prepare Code to Analyze) Set output variable 'artifacts-upload-name'
       id: set-artifacts-upload-name
       run: echo "artifacts-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-artifacts-input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
@@ -99,15 +95,9 @@ jobs:
       id: set-additional-maven-artifacts
       run: echo "additional-maven-artifacts=org.axonframework:axon-messaging:${{ env.AXON_FRAMEWORK_VERSION }},org.axonframework:axon-modelling:${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
 
-    - name: (Prepare Code to Analyze) Upload sources to analyze
-      if: success()
-      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-      with:
-        name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
-        path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source
-        include-hidden-files: true
-        if-no-files-found: error
-        retention-days: 1
+    - name: (Prepare Code to Analyze) Set output variable 'source-repository-branch'
+      id: set-source-repository-branch
+      run: echo "source-repository-branch=axon-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
 
     - name: (Prepare Code to Analyze) Upload artifacts to analyze
       if: success()
@@ -130,5 +120,7 @@ jobs:
       artifacts-upload-name: ${{ needs.prepare-code-to-analyze.outputs.artifacts-upload-name }}
       # Additional (duplicate) artifacts are only used here to test maven-artifacts parameter.
       maven-artifacts: ${{needs.prepare-code-to-analyze.outputs.additional-maven-artifacts}}
-      sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }}
+      source-repository: https://github.com/AxonFramework/AxonFramework.git
+      source-repository-branch: ${{ needs.prepare-code-to-analyze.outputs.source-repository-branch}}
+      source-repository-history-only: true
       jupyter-pdf: "false"

.github/workflows/public-analyze-code-graph.yml

@@ -35,6 +35,28 @@ on:
         required: false
         type: string
         default: ''
+      source-repository:
+        description: >
+          The URL of the source repository to analyze. For now, only GitHub repositories are supported.
+          This can be used instead of 'sources-upload-name' to directly analyze a repository without uploading artifacts first.
+          It can also be used in addition to 'sources-upload-name' to analyze both uploaded sources and a repository.
+        required: false
+        type: string
+        default: ''  
+      source-repository-branch:
+        description: >
+          The branch, tag or SHA of the source repository to checkout.
+          Default: default branch of the repository
+        required: false
+        type: string
+        default: ''
+      source-repository-history-only:
+        description: >
+          Whether to clone the source repository as a bare repository ("true") or not ("false", default).
+          Bare repositories do not have a working directory and are useful for git history analysis when the sources are not needed.
+        required: false
+        type: string
+        default: 'false'      
       ref:
         description: >
           The branch, tag or SHA of the code-graph-analysis-pipeline to checkout. 
@@ -95,9 +117,9 @@ jobs:
           python: 3.12
           miniforge: 24.9.0-0
     steps:
-      - name: Assure that either artifacts-upload-name or maven-artifacts or sources-upload-name is set
-        if: inputs.artifacts-upload-name == '' && inputs.maven-artifacts == ''  && inputs.sources-upload-name == ''
-        run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'maven-artifacts' or 'sources-upload-name'."; exit 1
+      - name: Assure that either artifacts-upload-name or maven-artifacts or sources-upload-name or source-repository is set
+        if: inputs.artifacts-upload-name == '' && inputs.maven-artifacts == ''  && inputs.sources-upload-name == '' && inputs.source-repository == ''
+        run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'maven-artifacts' or 'sources-upload-name' or 'source-repository'."; exit 1
       - name: Verify analysis-name only consists of characters safe for folder names
         run: |
           if [[ ! "${{ inputs.analysis-name }}" =~ ^[A-Za-z0-9._-]+$ ]]; then
@@ -176,6 +198,11 @@ jobs:
           name: ${{ inputs.sources-upload-name }}
           path: temp/${{ inputs.analysis-name }}/source/${{ inputs.analysis-name }}
 
+      - name: (Code Analysis Setup) Clone source repository for analysis
+        if: inputs.source-repository != ''
+        working-directory: temp/${{ inputs.analysis-name }}
+        run: ./../../scripts/cloneGitRepository.sh --url "${{ inputs.source-repository }}" --branch "${{ inputs.source-repository-branch }}" --history-only "${{ inputs.source-repository-history-only }}" --target "source/${{ inputs.analysis-name }}"
+
       - name: (Code Analysis Setup) Download artifacts for analysis
         if: inputs.artifacts-upload-name != ''
         uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5

INTEGRATION.md

@@ -37,6 +37,9 @@ The workflow parameters are as follows:
 - **sources-upload-name**: The name of the sources uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'source' directory for the analysis. It also supports sub-folders for multiple source code bases. This parameter is optional and defaults to an empty string.
 Please use 'include-hidden-files: true' if you also want to upload the git history.
 - **ref**: The branch, tag, or SHA of the code-graph-analysis-pipeline to checkout. This parameter is optional and defaults to "main".
+- **source-repository**: The URL of the source code repository to analyze. This parameter is optional and defaults to an empty string. If provided, it will be used to clone the repository for analysis instead of using the uploaded source code artifact. Currently. only public GitHub repositories are supported.
+- **source-repository-branch**: The branch of the source code repository to analyze. This parameter is optional and defaults to "main". It is only used if the 'source-repository' parameter is provided.
+- **source-repository-history-only**: If set to 'true', only the git history of the source code repository will be cloned for analysis. This parameter is optional and defaults to 'false'. It is only used if the 'source-repository' parameter is provided.
 - **analysis-arguments**: The arguments to pass to the analysis script. This parameter is optional and defaults to '--profile Neo4j-latest-low-memory'. You can find all available options in section [Command Line Options of COMMANDS.md/](./COMMANDS.md#command-line-options).
 - **typescript-scan-heap-memory**: The heap memory size in MB to use for the TypeScript code scans. This value is only used for the TypeScript code scans and is ignored for other scans. This parameter is optional and defaults to '4096'. It will set the environment variable `TYPESCRIPT_SCAN_HEAP_MEMORY` which leads to `NODE_OPTIONS` set to `--max-old-space-size=4096` for TypeScript scans. See [Questions and Answers of README.md](./README.md#thinking-questions--answers) for more information.
 

scripts/cloneGitRepository.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+
+# Provides safe-guarded (security checked parameters) git repository cloning.
+
+# Note: This script needs the path to target directory to clone the git repository to. It defaults to SOURCE_DIRECTORY ("source"). 
+# Note: This script needs git to be installed.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Overrideable Defaults
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
+
+# Local constants
+SCRIPT_NAME=$(basename "${0}")
+
+fail() {
+  local ERROR_COLOR='\033[0;31m' # red
+  local DEFAULT_COLOR='\033[0m'
+  local errorMessage="${1}"
+  echo -e "${ERROR_COLOR}${SCRIPT_NAME}: Error: ${errorMessage}${DEFAULT_COLOR}" >&2
+  exit 1
+}
+
+# Default and initial values for command line options
+url=""
+branch="main"
+history_only="false"
+target="${SOURCE_DIRECTORY}"
+dry_run="false"
+
+# Read command line options
+USAGE="${SCRIPT_NAME}: Usage: $0 --url <github-repository-url> --branch <branch-name> [--history-only <true|false>] [--target <clone directory>(default=source)]"
+
+while [ "$#" -gt "0" ]; do
+  key="$1"
+  case ${key} in
+    --url)
+      url="$2"
+      shift
+      ;;
+    --branch)
+      branch="$2"
+      shift
+      ;;
+    --history-only)
+      history_only="$2"
+      shift
+      ;;
+    --target)
+      target="$2"
+      shift
+      ;;
+    --dry-run)
+      dry_run="true"
+      ;;
+    *)
+      fail "Unknown option: ${key}"
+      echo "${USAGE}" >&2
+      exit 1
+  esac
+  shift
+done
+
+# --- Validate URL (mandatory)
+if [ -z "${url}" ] ; then
+  fail "The git repository URL (--url) must be provided."
+  echo "${USAGE}" >&2
+  exit 1
+fi
+case "${url}" in
+  https://github.com/*/*|https://github.com/*/*.git)
+    ;;
+  *)
+    fail "The source repository (--url) must be a valid GitHub repository URL."
+    ;;
+esac
+
+# --- Validate branch (mandatory, defaults to "main")
+if [ -z "${branch}" ] ; then
+  fail "The git repository branch (--branch) must be provided."
+  echo "${USAGE}" >&2
+  exit 1
+fi
+case "${branch}" in
+  *[\ ~^:?*[\]\\]*)
+    fail "The source repository branch contains invalid characters."
+    ;;
+esac
+
+# --- Validate history-only (mandatory, defaults to "false")
+case "${history_only}" in
+  true|false)
+    ;;
+  *)
+    fail "The source repository history-only option must be either 'true' or 'false'."
+    echo "${USAGE}" >&2
+    ;;
+esac
+
+# --- Validate target directory (mandatory, defaults to SOURCE_DIRECTORY)
+if [ -z "${target}" ] ; then
+  fail "The target directory (--target) ${target} must be provided." >&2
+  echo "${USAGE}" >&2
+  exit 1
+else
+  mkdir -p "${target}"
+fi
+
+if [ ${dry_run} = "true" ] ; then
+  echo "Dry run mode enabled. The following command(s) would be executed:" >&2
+fi
+
+# --- Clone the git repository
+bare_option=""
+bare_folder=""
+if [ "${history_only}" = "true" ]; then
+  bare_option="--bare"
+  bare_folder="/.git" # bare clones need the .git folder to be used as target
+fi
+
+if [ ${dry_run} = "true" ] ; then
+  echo "git clone ${bare_option} --single-branch ${url} --branch ${branch} ${target}${bare_folder}"
+  exit 0
+else
+  git clone ${bare_option} --single-branch  "${url}" --branch "${branch}" "${target}${bare_folder}"
+fi

scripts/downloader/downloadAxonFramework.sh

@@ -22,13 +22,19 @@ echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME_WITHOUT_EXTENSION=${SCRIPT_FILE
 echo "download${ANALYSIS_NAME}: ANALYSIS_NAME=${ANALYSIS_NAME}"
 
 # Read the first input argument containing the version(s) of the artifact(s)
-if [ "$#" -ne 1 ]; then
-  echo "Error (download${ANALYSIS_NAME}): Usage: $0 <version> (e.g. 4.9.3)" >&2
+if [ "$#" -eq 0 ]; then
+  echo "Error (download${ANALYSIS_NAME}): Usage: $0 <version> (e.g. 4.9.3) [--skip-clone]" >&2
   exit 1
 fi
 ARTIFACTS_VERSION=$1
 echo "download${ANALYSIS_NAME}: ARTIFACTS_VERSION=${ARTIFACTS_VERSION}"
 
+skipClone=false
+if [ "$2" = "--skip-clone" ]; then
+  skipClone=true
+  echo "download${ANALYSIS_NAME}: Git clone of source repository will be skipped."
+fi
+
 ## Get this "scripts/downloader" directory if not already set
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -55,7 +61,7 @@ source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g "${ARTIFACTS_GROUP}" -a "axo
 
 # Download the git history (bare clone without working tree) into the "source" folder.
 # This makes it possible to additionally import the git log into the graph
-if [ ! -d "${SOURCE_DIRECTORY}/AxonFramework-${ARTIFACTS_VERSION}/.git" ]; then
+if [ ! -d "${SOURCE_DIRECTORY}/AxonFramework-${ARTIFACTS_VERSION}/.git" ] && [ "${skipClone}" = false ]; then
   echo "download${ANALYSIS_NAME}: Getting bare git history of source code repository..."
   git clone --bare https://github.com/AxonFramework/AxonFramework.git --branch "axon-${ARTIFACTS_VERSION}" --single-branch "${SOURCE_DIRECTORY}/AxonFramework-${ARTIFACTS_VERSION}/.git"
 fi