Merge patch series "Rework requirements.txt files"

Tom Rini <trini@konsulko.com> says: A challenge we've run in to is making it easier for more people to use various python tools that we include in the tree. Part of the problem is that when we have a requirements.txt file, aside from the doc one we share with the kernel, I created it using "pip freeze". And while this might have been a best (or at least OK) practice at the time, that's no longer the case and is why our files have so many things in them. What this series does is create multiple files, one per project/tool and then has CI install them as needed. There's a few places here where this means that we update the requirements as well, but we keep a few big things where they are currently. This is because updating them introduces problems of their own and delaing with that would best be a follow up series. I've put this through GitLab and Azure to make sure everything is still going fine on both platforms. Link: https://lore.kernel.org/r/20250205000743.949790-1-trini@konsulko.com
2025-04-16 01:44:34 +00:00 · 2025-02-14 17:11:37 -06:00 · 2025-02-14 17:11:37 -06:00 · 8fe2c68c32
commit 8fe2c68c32
parent 455a003e8d fc9f3dd2e9
8 changed files with 63 additions and 54 deletions
--- a/.azure-pipelines.yml
+++ b/.azure-pipelines.yml
@ -134,8 +134,11 @@ stages:
          export USER=azure
          virtualenv -p /usr/bin/python3 /tmp/venv
          . /tmp/venv/bin/activate
-          pip install -r test/py/requirements.txt
-          pip install -r tools/buildman/requirements.txt
+          pip install -r test/py/requirements.txt \
+            -r tools/binman/requirements.txt \
+            -r tools/buildman/requirements.txt \
+            -r tools/patman/requirements.txt \
+            -r tools/u_boot_pylib/requirements.txt
          export UBOOT_TRAVIS_BUILD_DIR=/tmp/tools-only
          export PYTHONPATH=${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt
          export PATH=${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}
@ -163,9 +166,14 @@ stages:
      - script: |
          git config --global --add safe.directory $(work_dir)
          export USER=azure
-          pip install -r test/py/requirements.txt
-          pip install -r tools/buildman/requirements.txt
-          pip install asteval pylint==2.12.2 pyopenssl
+          virtualenv -p /usr/bin/python3 /tmp/venv
+          . /tmp/venv/bin/activate
+          pip install -r test/py/requirements.txt \
+            -r tools/binman/requirements.txt \
+            -r tools/buildman/requirements.txt \
+            -r tools/patman/requirements.txt \
+            -r tools/u_boot_pylib/requirements.txt \
+            asteval pylint==2.12.2 pyopenssl
          export PATH=${PATH}:~/.local/bin
          echo "[MASTER]" >> .pylintrc
          echo "load-plugins=pylint.extensions.docparams" >> .pylintrc
@ -265,7 +273,13 @@ stages:
          if [ -n "\${BUILD_ENV}" ]; then
              export \${BUILD_ENV};
          fi
-          pip install -r tools/buildman/requirements.txt
+          virtualenv -p /usr/bin/python3 /tmp/venv
+          . /tmp/venv/bin/activate
+          pip install -r tools/binman/requirements.txt \
+            -r tools/buildman/requirements.txt \
+            -r test/py/requirements.txt \
+            -r tools/u_boot_pylib/requirements.txt \
+            pytest-azurepipelines
          tools/buildman/buildman -o \${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e --board \${TEST_PY_BD} \${OVERRIDE}
          cp /opt/grub/grub_x86.efi \${UBOOT_TRAVIS_BUILD_DIR}/
          cp /opt/grub/grub_x64.efi \${UBOOT_TRAVIS_BUILD_DIR}/
@ -289,10 +303,6 @@ stages:
              /opt/coreboot/cbfstool \${UBOOT_TRAVIS_BUILD_DIR}/coreboot.rom remove -n fallback/payload;
              /opt/coreboot/cbfstool \${UBOOT_TRAVIS_BUILD_DIR}/coreboot.rom add-flat-binary -f \${UBOOT_TRAVIS_BUILD_DIR}/u-boot.bin -n fallback/payload -c LZMA -l 0x1110000 -e 0x1110000;
          fi
-          virtualenv -p /usr/bin/python3 /tmp/venv
-          . /tmp/venv/bin/activate
-          pip install -r test/py/requirements.txt
-          pip install pytest-azurepipelines
          export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:\${PATH}
          export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci
          # "\${var:+"-k \$var"}" expands to "" if \$var is empty, "-k \$var" if not
@ -582,7 +592,10 @@ stages:
          # make environment variables available as tests are running inside a container
          export BUILDMAN="${BUILDMAN}"
          git config --global --add safe.directory ${WORK_DIR}
-          pip install -r tools/buildman/requirements.txt
+          virtualenv -p /usr/bin/python3 /tmp/venv
+          . /tmp/venv/bin/activate
+          pip install -r tools/binman/requirements.txt \
+            -r tools/buildman/requirements.txt
          EOF
          cat << "EOF" >> build.sh
          if [[ "${BUILDMAN}" != "" ]]; then
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -55,6 +55,11 @@ stages:
        wget -O /tmp/fip.bin https://artifacts.codelinaro.org/artifactory/linaro-419-sbsa-ref/latest/tf-a/fip.bin;
        export BINMAN_INDIRS=/tmp;
      fi
+    # Prepare python environment
+    - virtualenv -p /usr/bin/python3 /tmp/venv;
+      . /tmp/venv/bin/activate;
+      pip install -r test/py/requirements.txt -r tools/binman/requirements.txt
+        -r tools/buildman/requirements.txt -r tools/u_boot_pylib/requirements.txt

  after_script:
    - cp -v /tmp/${TEST_PY_BD}/*.{html,css,xml} .
@ -90,9 +95,6 @@ stages:
        /opt/coreboot/cbfstool ${UBOOT_TRAVIS_BUILD_DIR}/coreboot.rom remove -n fallback/payload;
        /opt/coreboot/cbfstool ${UBOOT_TRAVIS_BUILD_DIR}/coreboot.rom add-flat-binary -f ${UBOOT_TRAVIS_BUILD_DIR}/u-boot.bin -n fallback/payload -c LZMA -l 0x1110000 -e 0x1110000;
      fi
-    - virtualenv -p /usr/bin/python3 /tmp/venv
-    - . /tmp/venv/bin/activate
-    - pip install -r test/py/requirements.txt
    # "${var:+"-k $var"}" expands to "" if $var is empty, "-k $var" if not
    - export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:${PATH};
      export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci;
@ -120,9 +122,13 @@ build all platforms in a single job:
  tags:
    - ${HOST}
  script:
+    # Prepare python environment
+    - virtualenv -p /usr/bin/python3 /tmp/venv;
+      . /tmp/venv/bin/activate;
+      pip install -r tools/binman/requirements.txt
+        -r tools/buildman/requirements.txt
    - ret=0;
      git config --global --add safe.directory "${CI_PROJECT_DIR}";
-      pip install -r tools/buildman/requirements.txt;
      ./tools/buildman/buildman -o /tmp -PEWM -x xtensa || ret=$?;
      if [[ $ret -ne 0 ]]; then
        ./tools/buildman/buildman -o /tmp -seP;
@ -179,8 +185,9 @@ Run binman, buildman, dtoc, Kconfig and patman testsuites:
      export USER=gitlab;
      virtualenv -p /usr/bin/python3 /tmp/venv;
      . /tmp/venv/bin/activate;
-      pip install -r test/py/requirements.txt;
-      pip install -r tools/buildman/requirements.txt;
+      pip install -r test/py/requirements.txt -r tools/binman/requirements.txt
+        -r tools/buildman/requirements.txt -r tools/patman/requirements.txt
+        -r tools/u_boot_pylib/requirements.txt;
      export UBOOT_TRAVIS_BUILD_DIR=/tmp/tools-only;
      export PYTHONPATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt";
      export PATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}";
@ -199,9 +206,11 @@ Run pylint:
  extends: .testsuites
  script:
    - git config --global --add safe.directory "${CI_PROJECT_DIR}"
-    - pip install -r test/py/requirements.txt
-    - pip install -r tools/buildman/requirements.txt
-    - pip install asteval pylint==2.12.2 pyopenssl
+    - virtualenv -p /usr/bin/python3 /tmp/venv
+    - . /tmp/venv/bin/activate
+    - pip install -r test/py/requirements.txt -r tools/binman/requirements.txt
+        -r tools/buildman/requirements.txt -r tools/patman/requirements.txt
+        -r tools/u_boot_pylib/requirements.txt asteval pylint==2.12.2 pyopenssl
    - export PATH=${PATH}:~/.local/bin
    - echo "[MASTER]" >> .pylintrc
    - echo "load-plugins=pylint.extensions.docparams" >> .pylintrc
--- a/test/py/requirements.txt
+++ b/test/py/requirements.txt
@ -1,30 +1,4 @@
-atomicwrites==1.4.1
-attrs==19.3.0
-concurrencytest==0.1.2
-coverage==6.2
-extras==1.0.0
 filelock==3.0.12
-fixtures==3.0.0
-importlib-metadata==0.23
-linecache2==1.0.0
-more-itertools==7.2.0
-packaging==24.1
-pbr==5.4.3
-pluggy==0.13.0
-py==1.11.0
-pycryptodomex==3.19.1
-pyelftools==0.27
-pygit2==1.13.3
-pyparsing==3.0.7
+pycryptodomex==3.21.0
 pytest==6.2.5
 pytest-xdist==2.5.0
-python-mimeparse==1.6.0
-python-subunit==1.3.0
-requests==2.32.3
-setuptools==70.3.0
-six==1.16.0
-testtools==2.3.0
-traceback2==1.4.0
-unittest2==1.1.0
-wcwidth==0.1.7
-zipp==3.19.2
--- a/tools/binman/requirements.txt
+++ b/tools/binman/requirements.txt
@ -0,0 +1,5 @@
+importlib_resources==6.5.2
+jsonschema==4.23.0
+pycryptodomex==3.21.0
+pyelftools==0.31
+yamllint==1.35.1
--- a/tools/buildman/requirements.txt
+++ b/tools/buildman/requirements.txt
@ -1,5 +1,2 @@
-coverage==6.2
-jsonschema==4.17.3
-pycryptodome==3.20
-pyyaml==6.0
-yamllint==1.26.3
+filelock==3.0.12
+importlib_resources==6.5.2
--- a/tools/docker/Dockerfile
+++ b/tools/docker/Dockerfile
@ -122,7 +122,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
 	python3 \
 	python3-dev \
 	python3-pip \
-	python3-pyelftools \
 	python3-sphinx \
 	python3-virtualenv \
 	rpm2cpio \
@ -308,12 +307,18 @@ USER uboot:uboot
 # COPY / ADD directives don't work as we need them to.
 RUN wget -O /tmp/pytest-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/test/py/requirements.txt
 RUN wget -O /tmp/sphinx-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/doc/sphinx/requirements.txt
+RUN wget -O /tmp/binman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/binman/requirements.txt
 RUN wget -O /tmp/buildman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/buildman/requirements.txt
+RUN wget -O /tmp/patman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/patman/requirements.txt
+RUN wget -O /tmp/u_boot_pylib-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/u_boot_pylib/requirements.txt
 RUN virtualenv -p /usr/bin/python3 /tmp/venv && \
 	. /tmp/venv/bin/activate && \
 	pip install -r /tmp/pytest-requirements.txt \
 		-r /tmp/sphinx-requirements.txt \
-		-r /tmp/buildman-requirements.txt && \
+		-r /tmp/binman-requirements.txt \
+		-r /tmp/buildman-requirements.txt \
+		-r /tmp/patman-requirements.txt \
+		-r /tmp/u_boot_pylib-requirements.txt && \
 	deactivate && \
 	rm -rf /tmp/venv /tmp/*-requirements.txt

--- a/tools/patman/requirements.txt
+++ b/tools/patman/requirements.txt
@ -0,0 +1,5 @@
+ConfigParser==7.1.0
+importlib_resources==6.5.2
+pygit2==1.13.3
+Requests==2.32.3
+setuptools==75.8.0
--- a/tools/u_boot_pylib/requirements.txt
+++ b/tools/u_boot_pylib/requirements.txt
@ -0,0 +1 @@
+concurrencytest==0.1.2