Update release process for Windows hosts

.gitignore

@@ -5,3 +5,4 @@ pandas-wheels
 pandas-feedstock
 pandas
 dist
+.vscode

Dockerfile

@@ -6,3 +6,18 @@ RUN apt-get update && \
 RUN conda update conda -y \
  && conda install -y conda-build conda-verify gcc_linux-64 gxx_linux-64 \
  && conda clean --all
+
+# ARG USER_ID
+# ARG GROUP_ID
+
+# RUN addgroup --gid $GROUP_ID user
+# RUN adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user
+
+# # https://github.com/ContinuumIO/docker-images/issues/151
+# RUN mkdir /opt/conda/envs/user && \
+#     chgrp user /opt/conda/pkgs && \
+#     chmod g+w /opt/conda/pkgs && \
+#     touch /opt/conda/pkgs/urls.txt && \
+#     chown user /opt/conda/envs/user /opt/conda/pkgs/urls.txt
+
+# USER user

Makefile

@@ -1,7 +1,7 @@
 # TO EDIT
-TAG ?= v1.1.0
+TAG ?= v1.4.3
+GH_USERNAME ?= simonjayhawkins
 
-GH_USERNAME ?= TomAugspurger
 PANDAS_VERSION=$(TAG:v%=%)
 PANDAS_BASE_VERSION=$(shell echo $(PANDAS_VERSION) | awk -F '.' '{OFS="."} { print $$1, $$2}')
 TARGZ=pandas-$(PANDAS_VERSION).tar.gz
@@ -14,15 +14,21 @@ SHELL := /bin/bash
 # -----------------------------------------------------------------------------
 
 init-repos:
-	git clone https://github.com/pandas-dev/pandas                   && git -C pandas           remote rename origin upstream && git -C pandas 		     remote add origin https://github.com/$(GH_USERNAME)/pandas
-	git clone https://github.com/conda-forge/pandas-feedstock        && git -C pandas-feedstock remote rename origin upstream && git -C pandas-feedstock remote add origin https://github.com/$(GH_USERNAME)/pandas-feedstock
-	git clone --recursive https://github.com/MacPython/pandas-wheels && git -C pandas-wheels    remote rename origin upstream && git -C pandas-wheels    remote add origin https://github.com/$(GH_USERNAME)/pandas-wheels
+	git clone https://github.com/pandas-dev/pandas \
+		&& git -C pandas remote rename origin upstream \
+		&& git -C pandas remote add origin https://github.com/$(GH_USERNAME)/pandas
+	git clone https://github.com/MacPython/pandas-wheels \
+		&& git -C pandas-wheels remote rename origin upstream \
+		&& git -C pandas-wheels remote add origin https://github.com/$(GH_USERNAME)/pandas-wheels
+	git clone https://github.com/conda-forge/pandas-feedstock \
+		&& git -C pandas-feedstock remote rename origin upstream \
+		&& git -C pandas-feedstock remote add origin https://github.com/$(GH_USERNAME)/pandas-feedstock
+
 
 update-repos:
-	git -C pandas checkout master           && git -C pandas pull
+	git -C pandas checkout main             && git -C pandas pull
 	git -C pandas-wheels checkout master    && git -C pandas-wheels pull
-	git -C pandas-feedstock checkout master && git -C pandas-feedstock pull
-	pushd pandas-wheels && git submodule update --recursive --remote && popd
+	git -C pandas-feedstock checkout main   && git -C pandas-feedstock pull
 
 # -----------------------------------------------------------------------------
 # Git Tag
@@ -37,25 +43,24 @@ tag:
 #  Builder Images
 # -----------------------------------------------------------------------------
 
-docker-image: pandas
+docker-image:
 	docker build -t pandas-build .
-
-
 docker-doc:
-	docker build -t pandas-docs -f docker-files/docs/Dockerfile .
+	DOCKER_BUILDKIT=1 docker build --progress=plain -t pandas-docs -f docker-files/docs/Dockerfile .
 
 
 # -----------------------------------------------------------------------------
 # sdist
 # -----------------------------------------------------------------------------
 
-pandas/dist/$(TARGZ):
+sdist:
 	docker run -it --rm \
 		--name=pandas-sdist-build \
 		-v ${CURDIR}/pandas:/pandas \
 		-v ${CURDIR}/scripts:/scripts \
 		pandas-build \
 		sh /scripts/build_sdist.sh
+	sudo chown -R $(shell id -u):$(shell id -g) pandas/dist/
 
 # -----------------------------------------------------------------------------
 # Tests
@@ -69,7 +74,7 @@ conda-test:
 		-v ${CURDIR}/pandas:/pandas \
 		-v ${CURDIR}/recipe:/recipe \
 		pandas-build \
-		sh -c "conda build --numpy=1.17.3 --python=3.8 /recipe --output-folder=/pandas/dist"
+		sh -c "conda build --numpy=1.17.3 --python=3.8 /recipe --output-folder=/pandas/dist -c conda-forge"
 
 pip-test: pandas/dist/$(TARGZ)
 	docker run -it --rm \
@@ -88,7 +93,7 @@ doc:
 		-v ${CURDIR}/pandas:/pandas \
 		-v ${CURDIR}/scripts/build-docs.sh:/build-docs.sh \
 		pandas-docs \
-		/build-docs.sh
+		/bin/bash
 
 
 upload-doc:
@@ -116,8 +121,9 @@ conda-forge:
 
 
 wheels:
-	rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp37m-linux_x86_64.whl
-	rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp37-cp37m-linux_x86_64.whl
+	rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp38m-linux_x86_64.whl
+	rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp38-cp38m-linux_x86_64.whl
+	rm -rf pandas/dist/pandas-$(PANDAS_VERSION)-cp38-cp38-linux_x86_64.whl
 	./scripts/wheels.sh $(TAG) $(GH_USERNAME)
 
 

README.md

@@ -1,10 +1,15 @@
-Release automation for pandas.
+# Release automation for pandas.
 
-**Windows users should follow the steps in `windows.md`**
+**Windows users should follow [these](./windows-wsl.md) steps.**
 
 ## Steps to a release
 
-- [  ] Manually update 
+The `environment.yml` contains the local dependencies. You'll
+also need docker.
+
+And fork pandas-wheels to your GitHub account.
+
+- [  ] Manually update
   - [  ] `TAG` in `Makefile`
   - [  ] `GH_USERNAME` in `Makefile`
 
@@ -14,11 +19,6 @@ If running for the first time be sure to initialize repos
 make init-repos
 ```
 
-The `environment.yml` contains the local dependencies. You'll
-also need docker.
-
-And fork pandas-feedstock and pandas-wheels to your GitHub account.
-
 ```
 # Update repos
 make update-repos
@@ -27,52 +27,54 @@ make update-repos
 make tag
 
 # Build the doc and test images
-make docker-image docker-doc
+make docker-image
+make docker-doc # seperate terminal - takes 10-20 mins to solve env
 
-# Build the sdist
-make pandas/dist/<>.tar.gz
+# Build the sdist (can do this in parallel with building docker-doc image above)
+make sdist
 
 # Final Pip and Conda tests. Do these in parallel.
-# You can optionally do make doc here as well
 make pip-test
 make conda-test
 
-# Push the tag. No going back now.
-make push-tag
-```
+# Docs. You can cheat and re-tag / rebuild these if needed.
+make doc
+./build-docs.sh
+exit
 
-Now manually create a release https://github.com/pandas-dev/pandas/releases
+# Visually check docs
+python -m http.server --directory pandas/doc/build/html/
+python -m http.server --directory pandas/doc/build/latex/
 
-Make sure to upload the sdist that's in `pandas/dist/` as the "binary".
-Conda-forge uses it.
+# Push the tag. No going back now.
+make push-tag
 
-On pandas you should also now create and tag a new branch, so
+# you may also need to create and tag a new branch, so for example:
 
-```sh
-git checkout -b <TAG>.x
-git push upstream <TAG>.x
-git checkout master
-git commit --allow-empty -m "Start <NEXT_TAG>"
-git tag -a v<NEXT_TAG>.dev0 -m 'DEV: Start <NEXT_TAG> cycle'
-git push upstream master --follow-tags
+cd pandas
+git checkout -b 1.4.x
+git push upstream 1.4.x
+git checkout main
+git commit --allow-empty -m "Start 1.5.0"
+git tag -a v1.5.0.dev0 -m 'DEV: Start 1.5.0'
+git push upstream main --follow-tags
+cd ..
 ```
 
-Start the binary builds.  **For Mac users** you may need to download the GNU version of sed before running this scripts via `brew install gnu-sed`
+Start the binary build.  **For Mac users** you may need to download the GNU version of sed before running this scripts via `brew install gnu-sed`
 
 ```
-# Binaries
-make conda-forge
 make wheels
 ```
 
-Open PRs for each of those.
+Open a PR at https://github.com/MacPython/pandas-wheels/pulls.
 
 Note that `make wheels` actually pushes a job to MacPython to produce wheels which we will download later.
 
-Docs. You can cheat and re-tag / rebuild these if needed.
+While the wheels are building, upload the built docs to the web server
 
 ```
-make doc
+make upload-doc
 ```
 
 Once the binaries finish, you'll need to manually upload the wheels to PyPI.
@@ -83,6 +85,17 @@ Assuming the job which `make wheels` triggered on MacPython completed successful
 make download-wheels
 ```
 
+if rc0, create version link for the docs to use in the github release announcement
+
+```
+make link-version
+```
+
+Now manually create a release https://github.com/pandas-dev/pandas/releases
+
+Make sure to upload the sdist that's in `pandas/dist/` as the "binary".
+Conda-forge uses it.
+
 Upload the wheels and sdist
 
 ```
@@ -91,17 +104,22 @@ make upload-pypi
 
 Finalize the docs
 
-```
-make upload-doc
-```
 
 To make sure /stable and the latest minor revision point to the new release run the following.
 
+**DO NOT DO THIS STEP FOR A RELEASE CANDIDATE**
+
 ```
 make link-stable
 make link-version
 ```
 
+if rc we need to manually create a PR since the conda-forge bot does not do this automatically on the dev branch. 
+
+```
+make conda-forge
+```
+
 goto announce.
 
 
@@ -111,16 +129,6 @@ goto announce.
 - [  ] Announce Mailing List
 - [  ] Announce Twitter
 
------
-
-## Initial Setup
-
-```
-# 1. Initialize Git Repositories
-make init-repos
-
-``````
-
 ---
 
 TODO:

docker-files/docs/Dockerfile

@@ -2,8 +2,11 @@ FROM continuumio/miniconda3:latest
 
 RUN apt-get update && apt-get install -y texlive-full xclip \
  && apt-get clean
+
 COPY pandas /pandas
-RUN conda env create --file=/pandas/environment.yml --name=pandas \
- && . activate pandas \
- && conda uninstall -y --force pandas \
- || true python setup.py build_ext -j 4
+
+RUN --mount=type=cache,target=/opt/conda/pkgs conda update conda -n base -y
+
+RUN --mount=type=cache,target=/opt/conda/pkgs conda install mamba -n base -c conda-forge -y
+
+RUN --mount=type=cache,target=/opt/conda/pkgs mamba env create --file=/pandas/environment.yml --name=pandas

recipe/meta.yaml

@@ -18,7 +18,7 @@ requirements:
   host:
     - python
     - pip
-    - cython >=0.29.13
+    - cython >=0.29.30, <3
     - numpy
   run:
     - python
@@ -29,10 +29,11 @@ requirements:
 test:
   requires:
     - pytest
+    - pytest-xdist
     - pytest-mock
     - hypothesis
   commands:
-    - python -c "import pandas; pandas.test(extra_args=['-m not clipboard', '--skip-slow', '--skip-network', '--skip-db'])"
+    - python -c "import pandas; pandas.test(extra_args=['-m not clipboard', '--skip-slow', '--skip-network', '--skip-db', '-n=2'])"
 
 about:
   home: http://pandas.pydata.org

scripts/build-docs.sh

@@ -4,17 +4,13 @@ set -e
 source activate pandas
 conda uninstall -y --force pandas ||:
 
-# this should have been built by pip-test
-# ugh that's 3.7, this is 3.6. boooooo
-# actually, can't really do that, since we upload the
-# dist folder to PyPI, and we don't want to upload this one...
-# python3 -m pip install --no-deps --no-index --find-links=/pandas/dist --only-binary=pandas pandas
-apt-get update && apt-get install -y build-essential
 cd /pandas
+
 python setup.py build_ext -i -j 4
 
 cd /pandas/doc
 
+./make.py clean
 ./make.py html
 ./make.py zip_html
 ./make.py latex_forced ||:

scripts/conda-forge.sh

@@ -22,7 +22,7 @@ git remote set-url origin https://github.com/${GH_USERNAME}/pandas-feedstock
 if [[ ${PANDAS_VERSION} == *"rc"* ]]; then
     git checkout dev
 else
-    git checkout master
+    git checkout main
 fi
 git pull upstream
 

scripts/pip_test.sh

@@ -2,10 +2,10 @@
 set -e
 
 # Cython is *not* required here.
-conda create -n pip-test -y python=3.7 numpy pytz python-dateutil pytest pytest-mock hypothesis nomkl
+conda create -n pip-test -y python=3.8 numpy pytz python-dateutil pytest pytest-xdist pytest-mock hypothesis nomkl
 
 source activate pip-test
 
 python3 -m pip wheel --no-deps --wheel-dir=/pandas/dist $1
 python3 -m pip install --no-deps --no-index --find-links=/pandas/dist --only-binary=pandas pandas
-python3 -c "import pandas; pandas.test(extra_args=['-m not clipboard', '--skip-slow', '--skip-network', '--skip-db'])"
+python3 -c "import pandas; pandas.test(extra_args=['-m not clipboard', '--skip-slow', '--skip-network', '--skip-db', '-n=2'])"