CHIA-597: Use poetry for package management (#11057)

* preliminary setup with poetry

* don't worry about locking the poetry env for the exploration right now

* recover black config in pyproject.toml

* install.sh for linting workflow

* pull chiavdf version from poetry in install-timelord.sh

* activate etc in upload and lint workflow

* debug poetry show output on macos

* update to clvm-tools 0.4.4

* add back setproctitle

* catchup deps

* lock

* allow 3.10

* debug

* make Install.ps1 use poetry as well

* -ErrorAction Ignore

* more todo about python version selection

* make install-timelord.sh modifications more posix compliant

* gitignore .penv* and .venv* as well

* remove requirements.penv.txt

* get artifact check requirements from poetry

* fix .gitignore for venv symlink

* separate out poetry setup

* :

* comment out code to be worked on

* setup poetry for dependency check

* ignore new poetry symlink

* chia_rs==0.1.4

* a few updates

* some missed updates `:[`

* update some types libs

* fix extras

* activate

* just use `python` somewhere

* for windows

* less symlinking for windows

* oops

* s

* more

* # shellcheck disable=SC2086

* activate action

* more python version

* lock up

* include ipython (for python >=3.8)

* add miniupnpc extra

* add poetry-dynamic-versioning

* type it out

* better match dev versioning

* catch up and delete setup.py again

* lock

* add pytest-cov

* .penv/bin/poetry

* .venv for windows activation

* update pytest-xdist

* install upnp extra

* upnp

* quote

* maybe update some deps

* mandatory and string

* oops

* slighty tidy

* debug pip list

* debug

* pybind11 maybe instead?

* remove pip install debugging

* debug without upnp nor pypi.chia.net

* chiavdf==1.0.8

* bring back upnp

* re-enable upnp

* diagnostic exception reraise

* .exe?

* diagnostics

* .cmd...?

* light tidy

* more cleanup

* make dependency check script not cwd dependent for finding the poetry env

* correct sh substitution

* lock

* coverage 7

* todo cleanup

* poetry update

* interesting

* windows too

* support non-editable install via scripts again

* oops

* again

* sheesh

* fixup

* more

* more

* Update dependencies

* various pre-commit linter updates

* add pyupgrade to extras list

* use priority=supplemental instead of secondary

* Add in some missing deps

* update setuptools and pyinstaller to match main

* venv->.venv

* lint fixes

* update a few packages for 3.12 support

* get zstd from chia resource for python 3.12

* Adjustments to dynamic versioning

* experiments with versioning

* test fixes

Signed-off-by: Earle Lowe <e.lowe@chia.net>

* fixup last commit

* include upnp as extra when installing on windows

* include mozilla-ca as a package for the cacert.pem file

* some windows tweaks

* use SPECPATH as anchor path in pyinstaller.spec file

* Add lxml as dev dependency

* Fixup typo in lxml version

* Update some packages to resolve GH vuln reports

* Update more packages to resolve GH vulns

* Ignore pylint license in depedency checker

* match installation of the dynamic version plugin on windows

* Fix issues with setting windows environment and use pip to install plugin per poetry docs recommendation

* properly apply non-editable option with -i

* use -f with ln to remove existing file as needed

* use shell=true for subprocess to help windows run chia if it's a .cmd

* Update .coveragerc

* use shutil.which() to find chia executable in a cross platform way

* mypy fixes

* Address conflicts for numpy based on python version

* Update lock file

* Remove troublesome reference to pypi source

* Add TODOs for manual install of poetry-dynamic-versioning plugin

* Use poetry dynamic versioning to figure out the version rather than setuptools_scm

* Update .github/workflows/reflow-version.yml

Co-authored-by: Kyle Altendorf <sda@fstab.net>

* Update .github/workflows/test-single.yml

Co-authored-by: Kyle Altendorf <sda@fstab.net>

* Update urllib3

* Attempts to get dynamic versioning to output something semver

* more versioning finangling

* lint pyproject.toml

* Update certifi

---------

Signed-off-by: Earle Lowe <e.lowe@chia.net>
Co-authored-by: Earle Lowe <e.lowe@chia.net>
Co-authored-by: Earle Lowe <30607889+emlowe@users.noreply.github.com>

install.sh

@@ -18,20 +18,20 @@ usage() {
   echo "${USAGE_TEXT}"
 }
 
-EXTRAS=
+EXTRAS='--extras upnp'
 PLOTTER_INSTALL=
-EDITABLE='-e'
+EDITABLE=1
 
 while getopts adilpsh flag; do
   case "${flag}" in
   # automated
   a) : ;;
   # development
-  d) EXTRAS=${EXTRAS}dev, ;;
+  d) EXTRAS="${EXTRAS} --extras dev" ;;
   # non-editable
-  i) EDITABLE='' ;;
+  i) EDITABLE= ;;
   # legacy keyring
-  l) EXTRAS=${EXTRAS}legacy-keyring, ;;
+  l) EXTRAS="${EXTRAS} --extras legacy-keyring" ;;
   p) PLOTTER_INSTALL=1 ;;
   # simple install
   s) : ;;
@@ -166,41 +166,23 @@ echo "OpenSSL version for Python is ${OPENSSL_VERSION_STRING}"
 if [ "$OPENSSL_VERSION_INT" -lt "269488367" ]; then
   echo "WARNING: OpenSSL versions before 3.0.2, 1.1.1n, or 1.0.2zd are vulnerable to CVE-2022-0778"
   echo "Your OS may have patched OpenSSL and not updated the version to 1.1.1n"
-  echo "We recommend updating to the latest version of OpenSSL available for your OS"
 fi
 
-# If version of `python` and "$INSTALL_PYTHON_VERSION" does not match, clear old version
-VENV_CLEAR=""
-if [ -e venv/bin/python ]; then
-  VENV_PYTHON_VER=$(venv/bin/python -V)
-  TARGET_PYTHON_VER=$($INSTALL_PYTHON_PATH -V)
-  if [ "$VENV_PYTHON_VER" != "$TARGET_PYTHON_VER" ]; then
-    echo "existing python version in venv is $VENV_PYTHON_VER while target python version is $TARGET_PYTHON_VER"
-    echo "Refreshing venv modules..."
-    VENV_CLEAR="--clear"
-  fi
-fi
-
-$INSTALL_PYTHON_PATH -m venv venv $VENV_CLEAR
+./setup-poetry.sh -c "${INSTALL_PYTHON_PATH}"
+.penv/bin/poetry env use "${INSTALL_PYTHON_PATH}"
+# TODO: Decide if this is needed or should be handled automatically in some way
+.penv/bin/pip install "poetry-dynamic-versioning[plugin]"
+# shellcheck disable=SC2086
+.penv/bin/poetry install ${EXTRAS}
+ln -s -f .venv venv
 if [ ! -f "activate" ]; then
   ln -s venv/bin/activate .
 fi
 
-EXTRAS=${EXTRAS%,}
-if [ -n "${EXTRAS}" ]; then
-  EXTRAS=[${EXTRAS}]
+if [ -z "$EDITABLE" ]; then
+  .venv/bin/python -m pip install --no-deps .
 fi
 
-# shellcheck disable=SC1091
-. ./activate
-# pip 20.x+ supports Linux binary wheels
-python -m pip install --upgrade pip
-python -m pip install wheel
-#if [ "$INSTALL_PYTHON_VERSION" = "3.8" ]; then
-# This remains in case there is a diversion of binary wheels
-python -m pip install --extra-index-url https://pypi.chia.net/simple/ miniupnpc==2.2.2
-python -m pip install ${EDITABLE} ."${EXTRAS}" --extra-index-url https://pypi.chia.net/simple/
-
 if [ -n "$PLOTTER_INSTALL" ]; then
   set +e
   PREV_VENV="$VIRTUAL_ENV"