diff --git a/.github/workflows/image_changes.sh b/.github/workflows/image_changes.sh index 8008f0bd786..6fa3bf50185 100755 --- a/.github/workflows/image_changes.sh +++ b/.github/workflows/image_changes.sh @@ -25,8 +25,6 @@ function github_ricj_callback() { show_changes_env+=( # Override the default locations of repositories. "SCRIPTS_REPO=." - "COREOS_OVERLAY_REPO=../coreos-overlay" - "PORTAGE_STABLE_REPO=../portage-stable" ) show_changes_params+=( # We may not have a tag handy, so we tell show-changes diff --git a/.github/workflows/update-metadata-glsa.yaml b/.github/workflows/update-metadata-glsa.yaml deleted file mode 100644 index b299785fee4..00000000000 --- a/.github/workflows/update-metadata-glsa.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: Keep GLSA metadata updated -on: - schedule: - - cron: '0 7 1 * *' - workflow_dispatch: - -jobs: - keep-glsa-metadata-updated: - runs-on: ubuntu-latest - steps: - - name: Check out scripts - uses: actions/checkout@v4 - with: - token: ${{ secrets.BOT_PR_TOKEN }} - - name: Update GLSA metadata - id: update-glsa-metadata - run: | - glsa_dir='sdk_container/src/third_party/portage-stable/metadata/glsa' - rm -rf "${glsa_dir}" - mkdir --parents "${glsa_dir}" - rsync --archive rsync://rsync.gentoo.org/gentoo-portage/metadata/glsa/* "${glsa_dir}" - todaydate=$(date +%Y-%m-%d) - echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}" - - name: Create pull request for main branch - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.BOT_PR_TOKEN }} - branch: buildbot/monthly-glsa-metadata-updates-${{steps.update-glsa-metadata.outputs.TODAYDATE }} - delete-branch: true - base: main - title: Monthly GLSA metadata ${{steps.update-glsa-metadata.outputs.TODAYDATE }} - body: Updated GLSA metadata - commit-message: "portage-stable/metadata: Monthly GLSA metadata updates" - author: Flatcar Buildbot - labels: main - signoff: true diff --git a/.github/workflows/update-portage-stable-packages-from-list.yaml b/.github/workflows/update-portage-stable-packages-from-list.yaml index 044b77af865..8a45e31f2bb 100644 --- a/.github/workflows/update-portage-stable-packages-from-list.yaml +++ b/.github/workflows/update-portage-stable-packages-from-list.yaml @@ -1,92 +1,53 @@ -name: Keep portage-stable packages updated +name: Keep Gentoo submodule updated on: schedule: - - cron: '0 7 * * 1' + - cron: '0 4 * * 0-4' workflow_dispatch: jobs: - keep-packages-updated: + keep-gentoo-updated: runs-on: ubuntu-latest steps: - name: Check out scripts - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: - token: ${{ secrets.GITHUB_TOKEN }} - path: ./scripts - - name: Check out Gentoo - uses: actions/checkout@v4 - with: - repository: gentoo/gentoo - path: gentoo - # Gentoo is quite a large repo, so limit ourselves to last - # quarter milion of commits. It is about two years worth of changes. - # Is is needed by the sync script to find out the hash of the last commit - # that made the changes to the package. - fetch-depth: 250000 - ref: master - - name: Check out build scripts - uses: actions/checkout@v4 - with: - repository: flatcar/flatcar-build-scripts - path: flatcar-build-scripts - - name: Update listed packages - id: update-listed-packages + token: ${{ secrets.BOT_PR_TOKEN }} + path: scripts + sparse-checkout: | + .gitmodules + .github/workflows/common.sh + sparse-checkout-cone-mode: false + - name: Update Gentoo submodule + id: update-gentoo + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" run: | - git config --global user.name "Flatcar Buildbot" - git config --global user.email "buildbot@flatcar-linux.org" - old_head=$(git -C scripts rev-parse HEAD) - packages_list=$(realpath scripts/.github/workflows/portage-stable-packages-list) - gentoo_repo=$(realpath gentoo) - build_scripts=$(realpath flatcar-build-scripts) - pushd scripts/sdk_container/src/third_party/portage-stable - while read -r package; do - if [[ ! -e "${package}" ]]; then - # If this happens, it means that the package was moved to overlay - # or dropped, the list ought to be updated. - echo "::warning title=${package}::Nonexistent package" - continue - fi - if [[ ! -e "${gentoo_repo}/${package}" ]]; then - # If this happens, it means that the package was obsoleted or moved - # in Gentoo. The obsoletion needs to be handled in the case-by-case - # manner, while move should be handled by doing the same move - # in portage-stable. The build should not break because of the move, - # because most likely it's already reflected in the profiles/updates - # directory. - echo "::warning title=${package}::Obsolete or moved package" - continue - fi - GENTOO_REPO="${gentoo_repo}" "${build_scripts}/sync-with-gentoo" "${package}" - done < <(grep '^[^#]' "${packages_list}") - popd - new_head=$(git -C scripts rev-parse HEAD) - updated=0 - if [[ "${new_head}" != "${old_head}" ]]; then - updated=1 + set -euo pipefail + source scripts/.github/workflows/common.sh + prepare_git_repo + submodule=sdk_container/src/third_party/gentoo + url=$(git -C "${WORK_SCRIPTS_DIR}" config --file .gitmodules --get "submodule.${submodule}.url") + branch=$(git -C "${WORK_SCRIPTS_DIR}" config --file .gitmodules --get "submodule.${submodule}.branch") + read -r latest_commit _ < <(git ls-remote --exit-code "${url}" "refs/heads/${branch}") + short_commit=${latest_commit:0:12} + git -C "${WORK_SCRIPTS_DIR}" update-index --cacheinfo "160000,${latest_commit},${submodule}" + if git -C "${WORK_SCRIPTS_DIR}" diff --cached --quiet -- "${submodule}"; then + echo 'UPDATE_NEEDED=0' >>"${GITHUB_OUTPUT}" + exit 0 fi - todaydate=$(date +%Y-%m-%d) - echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}" - echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}" - - name: Create pull request for main branch - uses: peter-evans/create-pull-request@v6 - if: steps.update-listed-packages.outputs.UPDATED == 1 + git -C "${WORK_SCRIPTS_DIR}" commit --signoff -m "gentoo: Update to ${short_commit}" + cleanup_repo + echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" + echo "LATEST_COMMIT=${latest_commit}" >>"${GITHUB_OUTPUT}" + echo "SHORT_COMMIT=${short_commit}" >>"${GITHUB_OUTPUT}" + - name: Create pull request + uses: peter-evans/create-pull-request@v8 + if: steps.update-gentoo.outputs.UPDATE_NEEDED == 1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.BOT_PR_TOKEN }} path: scripts - branch: buildbot/weekly-portage-stable-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }} + branch: buildbot/update-gentoo-submodule delete-branch: true - base: main - title: Weekly portage-stable package updates ${{steps.update-listed-packages.outputs.TODAYDATE }} - body: | - CI: TODO - - -- - - TODO: Changes. - - -- - - - [ ] changelog - - [ ] image diff - labels: main - draft: true + title: Update gentoo submodule to ${{ steps.update-gentoo.outputs.SHORT_COMMIT }} + body: Update `sdk_container/src/third_party/gentoo` to flatcar-hub/gentoo@${{ steps.update-gentoo.outputs.LATEST_COMMIT }}. + labels: auto-ci,jenkins-ci,main diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..cfedcf70075 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,5 @@ +[submodule "sdk_container/src/third_party/gentoo"] + path = sdk_container/src/third_party/gentoo + url = https://github.com/flatcar-hub/gentoo.git + branch = master + shallow = true diff --git a/PREFIX.md b/PREFIX.md index d0935b9e18b..872b3c6a31f 100644 --- a/PREFIX.md +++ b/PREFIX.md @@ -1,3 +1,11 @@ +# !!! OUTDATED. This needs to be ported to match the changes after the Gentoo move !!! + +If anything, working with prefixes should be a lot easier now. +But the below is outdated and will not work. +`prefix-overlay` has been removed too, as it duplicated ebuilds now available via `gentoo`. + +# !!! OUTDATED. This needs to be ported to match the changes after the Gentoo move !!! + # Prefix - build portable, distro-independent apps **!!! NOTE: Prefix support in the Flatcar SDK is EXPERIMENTAL at this time !!!** diff --git a/README.md b/README.md index fa843f0d230..9264ff2b91e 100644 --- a/README.md +++ b/README.md @@ -28,18 +28,17 @@ The SDK can be used to # Using the scripts repository The repository is meant to be the entry point for Flatcar builds and development. -Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [portage-stable](sdk_container/src/third_party/portage-stable/): +Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [gentoo](sdk_container/src/third_party/gentoo/): ``` scripts +--sdk_container +---------src +--third_party +------coreos-overlay - +------portage-stable + +------gentoo ``` -`portage-stable` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions). -Consider it a small sub-set of Gentoo. +`gentoo` is a git submodule, pointing to Flatcar's fork of the main upstream Gentoo package repository. That fork is generally aligned with Gentoo and only occasionally diverges into alternative branches for brief periods to apply fixes shortly before upstream does. It includes far more packages than Flatcar actually uses. `coreos-overlay` contains significantly modified or even entirely self-written ebuilds. @@ -66,7 +65,7 @@ While work on a native ARM64 native SDK is ongoing, it's unfortunately not ready The container can be run in one of two ways - "standalone", or integrated with the [scripts](https://github.com/flatcar/scripts) repo: * Standalone mode will use no host volumes and will allow you to play with the SDK in a sandboxed throw-away environment. In standalone mode, you interface with Docker directly to use the SDK container. -* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar. +* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the gentoo and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar. ## Standalone mode @@ -96,7 +95,7 @@ To start a container in privileged mode with `/dev` available use: This is the preferred mode of working with the SDK. Interaction with the container happens via wrapper scripts from the scripts repository. -Both the host's scripts repo as well as the ebuild paths (portage-stable and coreos-overlay) are made available in the container, allowing for work on these directly. +Both the host's scripts repo as well as the ebuild directories (gentoo and coreos-overlay) are made available in the container, allowing for work on these directly. The wrapper scripts will re-use existing containers instead of creating new ones to preserve your work in the container, enabling consistency. To clone the scripts repo and pick a version: diff --git a/bootstrap_sdk_container b/bootstrap_sdk_container index c197787179b..024a17303bb 100755 --- a/bootstrap_sdk_container +++ b/bootstrap_sdk_container @@ -9,9 +9,10 @@ set -eu cd $(dirname "$0") source sdk_lib/sdk_container_common.sh +unset seed_tarball +bootstrap_args=() seed_version="" target_version="" -logdir='' declare -a cleanup @@ -28,10 +29,10 @@ usage() { echo " The SDK container will be pulled and the tarball" echo " downloaded if necessary." echo " - SDK version number (e.g. '3027.0.0') of the new SDK." + echo " --seed_tarball - Optional path or URL to override the default seed tarball." echo " -x - For each resource generated during build (container etc.)" echo " add a cleanup line to